From c58f81787e7c2a985a167c15db9fd9ca5fed6c35 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 12:23:29 -0500 Subject: [PATCH 001/168] fix(wallet-toolbox): verify overlay identity evidence --- docs/packages/wallet/wallet-toolbox-client.md | 2 +- docs/packages/wallet/wallet-toolbox-mobile.md | 2 +- docs/packages/wallet/wallet-toolbox.md | 2 +- docs/reference/package-api-migrations.md | 84 ++-- docs/reference/stack-facts.md | 6 +- governance/package-release-notes.json | 12 +- governance/repository-health/baselines.json | 6 +- packages/wallet/wallet-toolbox/CHANGELOG.md | 11 + packages/wallet/wallet-toolbox/README.md | 20 +- .../wallet/wallet-toolbox/client/README.md | 7 + .../wallet/wallet-toolbox/client/package.json | 2 +- .../client/platform-budget.json | 8 +- .../docs/identity-verification.md | 106 +++++ .../wallet/wallet-toolbox/mobile/README.md | 7 + .../wallet/wallet-toolbox/mobile/package.json | 2 +- .../mobile/platform-budget.json | 6 +- packages/wallet/wallet-toolbox/package.json | 2 +- packages/wallet/wallet-toolbox/src/Wallet.ts | 74 ++-- .../Wallet.identityVerification.test.ts | 176 ++++++++ .../fixtures/identity-verification.json | 4 + .../identityVerification.fixtures.ts | 149 +++++++ .../__tests__/identityVerification.test.ts | 396 ++++++++++++++++++ .../src/utility/identityUtils.ts | 91 +++- .../src/utility/verifyOverlayOutput.ts | 79 ++++ 24 files changed, 1133 insertions(+), 121 deletions(-) create mode 100644 packages/wallet/wallet-toolbox/docs/identity-verification.md create mode 100644 packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts create mode 100644 packages/wallet/wallet-toolbox/src/utility/__tests__/fixtures/identity-verification.json create mode 100644 packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.fixtures.ts create mode 100644 packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.test.ts create mode 100644 packages/wallet/wallet-toolbox/src/utility/verifyOverlayOutput.ts diff --git a/docs/packages/wallet/wallet-toolbox-client.md b/docs/packages/wallet/wallet-toolbox-client.md index d5496eb82..c582ce5f3 100644 --- a/docs/packages/wallet/wallet-toolbox-client.md +++ b/docs/packages/wallet/wallet-toolbox-client.md @@ -3,7 +3,7 @@ id: pkg-wallet-toolbox-client title: '@bsv/wallet-toolbox-client' kind: package domain: wallet -version: '2.11.0' +version: '2.11.1' last_updated: '2026-08-31' last_verified: '2026-08-31' review_cadence_days: 30 diff --git a/docs/packages/wallet/wallet-toolbox-mobile.md b/docs/packages/wallet/wallet-toolbox-mobile.md index beb53276c..b01fad0ce 100644 --- a/docs/packages/wallet/wallet-toolbox-mobile.md +++ b/docs/packages/wallet/wallet-toolbox-mobile.md @@ -3,7 +3,7 @@ id: pkg-wallet-toolbox-mobile title: '@bsv/wallet-toolbox-mobile' kind: package domain: wallet -version: '2.11.0' +version: '2.11.1' last_updated: '2026-08-31' last_verified: '2026-08-31' review_cadence_days: 30 diff --git a/docs/packages/wallet/wallet-toolbox.md b/docs/packages/wallet/wallet-toolbox.md index 5f5487c55..583981ec0 100644 --- a/docs/packages/wallet/wallet-toolbox.md +++ b/docs/packages/wallet/wallet-toolbox.md @@ -4,7 +4,7 @@ title: '@bsv/wallet-toolbox' kind: package domain: wallet npm: '@bsv/wallet-toolbox' -version: '2.11.0' +version: '2.11.1' last_updated: '2026-08-31' last_verified: '2026-08-31' review_cadence_days: 30 diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 995d1e809..7c20e6db3 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -23,42 +23,42 @@ and clean-consumer tests remain the executable type authority. ## Current release boundary -| Package | npm baseline | Source | Candidate | API | Migration | -| --------------------------------- | ------------ | -------- | --------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `@bsv/402-pay` | `0.2.1` | `0.2.5` | patch | [API and usage](../packages/middleware/402-pay.md) | No consumer migration is required; client and server exports, payment protocol behavior, and runtime defaults are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/air-gap` | `0.0.0` | `0.1.2` | minor | [API and usage](../packages/helpers/air-gap.md) | No consumer migration is required; this is the first published release of a new package with no prior public API. The experimental pre-release framing that circulated on the unmerged feature branch is not accepted by the v1 decoder. | -| `@bsv/amountinator` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/helpers/amountinator.md) | No consumer migration is required; this is a backward-compatible patch candidate. | -| `@bsv/auth` | `0.1.1` | `0.1.4` | patch | [API and usage](../packages/middleware/auth.md) | No consumer migration is required; authentication APIs, wire behavior, and runtime defaults are unchanged. | -| `@bsv/auth-express-middleware` | `2.2.0` | `2.2.3` | patch | [API and usage](../packages/middleware/auth-express-middleware.md) | No API migration is required. Upgrade to @bsv/sdk 2.4.1 or later for the shared byte-boundary contract. Generic signed application-body canonicalization remains unchanged so old and new peers verify identical bytes. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/authsocket` | `2.1.1` | `2.1.7` | patch | [API and usage](../packages/messaging/authsocket.md) | No API migration is required. Existing event data, including numeric-key objects under byte-like names, is unchanged; typed payment protocols recover historical byte objects at their explicit fields. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/authsocket-client` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/messaging/authsocket-client.md) | No API migration is required. Existing event data, including numeric-key objects under byte-like names, is unchanged; typed payment protocols recover historical byte objects at their explicit fields. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | -| `@bsv/btms` | `1.1.1` | `1.2.2` | minor | [API and usage](../packages/wallet/btms.md) | Existing local, mainnet, testnet, and number-array behavior is unchanged. TTN consumers select networkPreset teratestnet; all consumers should upgrade to @bsv/sdk 2.4.1 or later for byte-boundary compatibility. | -| `@bsv/btms-permission-module` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/wallet/btms-permission-module.md) | No consumer migration is required; permission-module APIs and token semantics are unchanged. | -| `@bsv/chirp` | `0.0.0` | `0.1.1` | minor | [API and usage](../packages/network/chirp.md) | No consumer migration is required; this is the first release of a new additive package. Existing @bsv/sdk StorageUploader, StorageDownloader, StorageUtils, UHRP identifiers, overlays, and server routes remain unchanged. BRC-167 remains authoritative if the implementation and standard differ. | -| `@bsv/did` | `0.2.1` | `0.2.5` | patch | [API and usage](../packages/helpers/did.md) | No consumer migration is required; DID APIs, encodings, credential behavior, and supported import forms are unchanged. | -| `@bsv/did-client` | `1.2.1` | `1.3.1` | minor | [API and usage](../packages/helpers/did-client.md) | Existing local, mainnet, and testnet behavior is unchanged. TTN consumers select networkPreset teratestnet and use @bsv/sdk 2.4 or later. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | -| `@bsv/ecpm-permission-module` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/wallet/ecpm-permission-module.md) | No existing consumer migration is required; this is the first release. Wallet hosts register the module under the ecpm scheme and supply their ordinary key deriver, authorization handler, and optional privileged key provider. | -| `@bsv/fund-wallet` | `1.4.1` | `1.4.4` | patch | [API and usage](../packages/helpers/fund-wallet.md) | No consumer migration is required; wallet funding APIs and transaction behavior are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | -| `@bsv/message-box-client` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | -| `@bsv/overlay` | `2.2.1` | `2.3.1` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.1` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-express` | `2.5.0` | `2.6.1` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-topics` | `1.6.10` | `1.7.1` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. | -| `@bsv/paymail` | `2.4.2` | `2.4.7` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. Consumers of the former bundled Money Button or Tokenized specification documents must follow the authoritative links in docs/specs/README.md. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/payment-express-middleware` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/sdk` | `2.4.0` | `2.4.2` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. | -| `@bsv/simple` | `0.4.1` | `0.5.3` | minor | [API and usage](../packages/helpers/simple.md) | Existing overlay configurations and number-array behavior are unchanged. TTN consumers select network teratestnet; all consumers should upgrade to @bsv/sdk 2.4.2 or later. The Message Box Client dependency advance requires no helper API migration and preserves the default HTTP and live-socket behavior. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/. | -| `@bsv/templates` | `1.9.1` | `1.10.1` | minor | [API and usage](../packages/helpers/templates.md) | No existing consumer migration is required; existing template APIs and generated scripts are unchanged. New R1K1Wallet consumers await lock(), retain each private 32-byte salt, and provide a PIV signer that signs the supplied digest directly without hashing it again. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/teranode-listener` | `1.1.1` | `1.1.5` | patch | [API and usage](../packages/network/teranode-listener.md) | No consumer migration is required; listener APIs, topics, and network configuration are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/verifast` | `0.3.0` | `0.3.5` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. Keep THIRD_PARTY_NOTICES.md and LICENSES/ with every JavaScript and WebAssembly distribution. | -| `@bsv/wallet-helper` | `0.1.1` | `0.1.7` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/wallet-relay` | `0.2.2` | `0.3.6` | minor | [API and usage](../packages/wallet/wallet-relay.md) | No wallet RPC migration is required; upgrade to @bsv/sdk 2.4.1 or later. Existing relay sessions and number arrays remain valid, and host applications continue to provide their matching Express runtime and type graph. | -| `@bsv/wallet-toolbox` | `2.10.4` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/. | -| `@bsv/wallet-toolbox-client` | `2.10.4` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. | -| `@bsv/wallet-toolbox-mobile` | `2.10.4` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. | -| `create-bsv-app` | `1.0.2` | `1.1.1` | minor | [API and usage](../packages/helpers/create-bsv-app.md) | Existing mainnet and testnet scaffolds are unchanged. New TTN projects pass --network ttn or select TerraTestNet in the configurator. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| Package | npm baseline | Source | Candidate | API | Migration | +| --------------------------------- | ------------ | -------- | --------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `@bsv/402-pay` | `0.2.1` | `0.2.5` | patch | [API and usage](../packages/middleware/402-pay.md) | No consumer migration is required; client and server exports, payment protocol behavior, and runtime defaults are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/air-gap` | `0.0.0` | `0.1.2` | minor | [API and usage](../packages/helpers/air-gap.md) | No consumer migration is required; this is the first published release of a new package with no prior public API. The experimental pre-release framing that circulated on the unmerged feature branch is not accepted by the v1 decoder. | +| `@bsv/amountinator` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/helpers/amountinator.md) | No consumer migration is required; this is a backward-compatible patch candidate. | +| `@bsv/auth` | `0.1.1` | `0.1.4` | patch | [API and usage](../packages/middleware/auth.md) | No consumer migration is required; authentication APIs, wire behavior, and runtime defaults are unchanged. | +| `@bsv/auth-express-middleware` | `2.2.0` | `2.2.3` | patch | [API and usage](../packages/middleware/auth-express-middleware.md) | No API migration is required. Upgrade to @bsv/sdk 2.4.1 or later for the shared byte-boundary contract. Generic signed application-body canonicalization remains unchanged so old and new peers verify identical bytes. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/authsocket` | `2.1.1` | `2.1.7` | patch | [API and usage](../packages/messaging/authsocket.md) | No API migration is required. Existing event data, including numeric-key objects under byte-like names, is unchanged; typed payment protocols recover historical byte objects at their explicit fields. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/authsocket-client` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/messaging/authsocket-client.md) | No API migration is required. Existing event data, including numeric-key objects under byte-like names, is unchanged; typed payment protocols recover historical byte objects at their explicit fields. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | +| `@bsv/btms` | `1.1.1` | `1.2.2` | minor | [API and usage](../packages/wallet/btms.md) | Existing local, mainnet, testnet, and number-array behavior is unchanged. TTN consumers select networkPreset teratestnet; all consumers should upgrade to @bsv/sdk 2.4.1 or later for byte-boundary compatibility. | +| `@bsv/btms-permission-module` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/wallet/btms-permission-module.md) | No consumer migration is required; permission-module APIs and token semantics are unchanged. | +| `@bsv/chirp` | `0.0.0` | `0.1.1` | minor | [API and usage](../packages/network/chirp.md) | No consumer migration is required; this is the first release of a new additive package. Existing @bsv/sdk StorageUploader, StorageDownloader, StorageUtils, UHRP identifiers, overlays, and server routes remain unchanged. BRC-167 remains authoritative if the implementation and standard differ. | +| `@bsv/did` | `0.2.1` | `0.2.5` | patch | [API and usage](../packages/helpers/did.md) | No consumer migration is required; DID APIs, encodings, credential behavior, and supported import forms are unchanged. | +| `@bsv/did-client` | `1.2.1` | `1.3.1` | minor | [API and usage](../packages/helpers/did-client.md) | Existing local, mainnet, and testnet behavior is unchanged. TTN consumers select networkPreset teratestnet and use @bsv/sdk 2.4 or later. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | +| `@bsv/ecpm-permission-module` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/wallet/ecpm-permission-module.md) | No existing consumer migration is required; this is the first release. Wallet hosts register the module under the ecpm scheme and supply their ordinary key deriver, authorization handler, and optional privileged key provider. | +| `@bsv/fund-wallet` | `1.4.1` | `1.4.4` | patch | [API and usage](../packages/helpers/fund-wallet.md) | No consumer migration is required; wallet funding APIs and transaction behavior are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | +| `@bsv/message-box-client` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | +| `@bsv/overlay` | `2.2.1` | `2.3.1` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.1` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/overlay-express` | `2.5.0` | `2.6.1` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/overlay-topics` | `1.6.10` | `1.7.1` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. | +| `@bsv/paymail` | `2.4.2` | `2.4.7` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. Consumers of the former bundled Money Button or Tokenized specification documents must follow the authoritative links in docs/specs/README.md. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/payment-express-middleware` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/sdk` | `2.4.0` | `2.4.2` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. | +| `@bsv/simple` | `0.4.1` | `0.5.3` | minor | [API and usage](../packages/helpers/simple.md) | Existing overlay configurations and number-array behavior are unchanged. TTN consumers select network teratestnet; all consumers should upgrade to @bsv/sdk 2.4.2 or later. The Message Box Client dependency advance requires no helper API migration and preserves the default HTTP and live-socket behavior. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/. | +| `@bsv/templates` | `1.9.1` | `1.10.1` | minor | [API and usage](../packages/helpers/templates.md) | No existing consumer migration is required; existing template APIs and generated scripts are unchanged. New R1K1Wallet consumers await lock(), retain each private 32-byte salt, and provide a PIV signer that signs the supplied digest directly without hashing it again. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/teranode-listener` | `1.1.1` | `1.1.5` | patch | [API and usage](../packages/network/teranode-listener.md) | No consumer migration is required; listener APIs, topics, and network configuration are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/verifast` | `0.3.0` | `0.3.5` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. Keep THIRD_PARTY_NOTICES.md and LICENSES/ with every JavaScript and WebAssembly distribution. | +| `@bsv/wallet-helper` | `0.1.1` | `0.1.7` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/wallet-relay` | `0.2.2` | `0.3.6` | minor | [API and usage](../packages/wallet/wallet-relay.md) | No wallet RPC migration is required; upgrade to @bsv/sdk 2.4.1 or later. Existing relay sessions and number arrays remain valid, and host applications continue to provide their matching Express runtime and type graph. | +| `@bsv/wallet-toolbox` | `2.10.4` | `2.11.1` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. | +| `@bsv/wallet-toolbox-client` | `2.10.4` | `2.11.1` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. | +| `@bsv/wallet-toolbox-mobile` | `2.10.4` | `2.11.1` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. | +| `create-bsv-app` | `1.0.2` | `1.1.1` | minor | [API and usage](../packages/helpers/create-bsv-app.md) | Existing mainnet and testnet scaffolds are unchanged. New TTN projects pass --network ttn or select TerraTestNet in the configurator. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | `none` means the source manifest matches the recorded npm baseline. Any other value is an unpublished candidate. Publication, tags, releases, registry @@ -520,8 +520,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox.md](../packages/wallet/wallet-toolbox.md) - Source: [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) -- Release note: Adds the built-in BRC-177 noSend-expiry reference implementation with exact prefunding, durable pre-signed reclaim, atomic active-storage monitoring, backoff-controlled recovery, cross-device lifecycle synchronization, and proof-finalized race handling. Adds opt-in prepared BEEF storage for Knex-backed normal createAction funding: verified, checksummed proof closures are persisted after foreground completion and reused on later hits, while broad lookups, misses, and cache failures retain the canonical path. Reads, writes, bounded queueing, and gradual backfill default off; reorganizations stale derived rows and fence in-flight cross-process writes with a database proof epoch. Also adds the optional semantic handleRequest hook for BRC-98/99/111 permission modules, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes, retains BRC-95/BRC-100 compatibility and stable bounded pagination, removes the obsolete JSight application bundle, and preserves the earlier Open BSV grant. -- Migration: Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/. +- Release note: Adds the built-in BRC-177 noSend-expiry reference implementation with exact prefunding, durable pre-signed reclaim, atomic active-storage monitoring, backoff-controlled recovery, cross-device lifecycle synchronization, and proof-finalized race handling. Adds opt-in prepared BEEF storage for Knex-backed normal createAction funding: verified, checksummed proof closures are persisted after foreground completion and reused on later hits, while broad lookups, misses, and cache failures retain the canonical path. Reads, writes, bounded queueing, and gradual backfill default off; reorganizations stale derived rows and fence in-flight cross-process writes with a database proof epoch. Also adds the optional semantic handleRequest hook for BRC-98/99/111 permission modules, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes, retains BRC-95/BRC-100 compatibility and stable bounded pagination, removes the obsolete JSight application bundle, and preserves the earlier Open BSV grant. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. +- Migration: Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ---------------------------------------------------- | -------------------------- | @@ -534,8 +534,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-client.md](../packages/wallet/wallet-toolbox-client.md) - Source: [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) -- Release note: Adds the built-in browser BRC-177 noSend-expiry signer, IndexedDB schema version 5 lifecycle state, remote storage capability negotiation, default monitor coordination, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while IndexedDB remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, stable IndexedDB totals, current browser compatibility fixes, and earlier Open BSV grants. -- Migration: Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. +- Release note: Adds the built-in browser BRC-177 noSend-expiry signer, IndexedDB schema version 5 lifecycle state, remote storage capability negotiation, default monitor coordination, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while IndexedDB remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, stable IndexedDB totals, current browser compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. +- Migration: Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | @@ -546,8 +546,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-mobile.md](../packages/wallet/wallet-toolbox-mobile.md) - Source: [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) -- Release note: Adds the built-in mobile BRC-177 noSend-expiry signer, remote storage capability negotiation, default-monitor ownership coordination across restarts and devices, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback for React Native runtimes without WebAssembly, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while mobile storage remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, current mobile compatibility fixes, and earlier Open BSV grants. -- Migration: Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. +- Release note: Adds the built-in mobile BRC-177 noSend-expiry signer, remote storage capability negotiation, default-monitor ownership coordination across restarts and devices, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback for React Native runtimes without WebAssembly, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while mobile storage remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, current mobile compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. +- Migration: Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 13c442ae0..45b71107d 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -68,9 +68,9 @@ authorized release action. | wallet | `@bsv/btms-permission-module` | `1.1.4` | node-library | node-esm | node | `>=22` | [packages/wallet/btms-permission-module](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms-permission-module) | | wallet | `@bsv/ecpm-permission-module` | `0.1.0` | browser-library | browser-bundler, browser-esm, node-esm | browser, node | `>=22` | [packages/wallet/ecpm-permission-module](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/ecpm-permission-module) | | wallet | `@bsv/wallet-relay` | `0.3.6` | cli-library | browser-bundler, browser-esm, cli, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/ts-wallet-relay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/ts-wallet-relay) | -| wallet | `@bsv/wallet-toolbox` | `2.11.0` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | -| wallet | `@bsv/wallet-toolbox-client` | `2.11.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | -| wallet | `@bsv/wallet-toolbox-mobile` | `2.11.0` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | +| wallet | `@bsv/wallet-toolbox` | `2.11.1` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | +| wallet | `@bsv/wallet-toolbox-client` | `2.11.1` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | +| wallet | `@bsv/wallet-toolbox-mobile` | `2.11.1` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | ## Standalone infrastructure manifests diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index fb190dd4d..eccf304bb 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -217,22 +217,22 @@ "name": "@bsv/wallet-toolbox", "publishedVersion": "2.10.4", "releaseType": "minor", - "summary": "Adds the built-in BRC-177 noSend-expiry reference implementation with exact prefunding, durable pre-signed reclaim, atomic active-storage monitoring, backoff-controlled recovery, cross-device lifecycle synchronization, and proof-finalized race handling. Adds opt-in prepared BEEF storage for Knex-backed normal createAction funding: verified, checksummed proof closures are persisted after foreground completion and reused on later hits, while broad lookups, misses, and cache failures retain the canonical path. Reads, writes, bounded queueing, and gradual backfill default off; reorganizations stale derived rows and fence in-flight cross-process writes with a database proof epoch. Also adds the optional semantic handleRequest hook for BRC-98/99/111 permission modules, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes, retains BRC-95/BRC-100 compatibility and stable bounded pagination, removes the obsolete JSight application bundle, and preserves the earlier Open BSV grant.", - "migration": "Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/." + "summary": "Adds the built-in BRC-177 noSend-expiry reference implementation with exact prefunding, durable pre-signed reclaim, atomic active-storage monitoring, backoff-controlled recovery, cross-device lifecycle synchronization, and proof-finalized race handling. Adds opt-in prepared BEEF storage for Knex-backed normal createAction funding: verified, checksummed proof closures are persisted after foreground completion and reused on later hits, while broad lookups, misses, and cache failures retain the canonical path. Reads, writes, bounded queueing, and gradual backfill default off; reorganizations stale derived rows and fence in-flight cross-process writes with a database proof epoch. Also adds the optional semantic handleRequest hook for BRC-98/99/111 permission modules, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes, retains BRC-95/BRC-100 compatibility and stable bounded pagination, removes the obsolete JSight application bundle, and preserves the earlier Open BSV grant. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use.", + "migration": "Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness." }, { "name": "@bsv/wallet-toolbox-client", "publishedVersion": "2.10.4", "releaseType": "minor", - "summary": "Adds the built-in browser BRC-177 noSend-expiry signer, IndexedDB schema version 5 lifecycle state, remote storage capability negotiation, default monitor coordination, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while IndexedDB remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, stable IndexedDB totals, current browser compatibility fixes, and earlier Open BSV grants.", - "migration": "Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/." + "summary": "Adds the built-in browser BRC-177 noSend-expiry signer, IndexedDB schema version 5 lifecycle state, remote storage capability negotiation, default monitor coordination, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while IndexedDB remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, stable IndexedDB totals, current browser compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use.", + "migration": "Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness." }, { "name": "@bsv/wallet-toolbox-mobile", "publishedVersion": "2.10.4", "releaseType": "minor", - "summary": "Adds the built-in mobile BRC-177 noSend-expiry signer, remote storage capability negotiation, default-monitor ownership coordination across restarts and devices, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback for React Native runtimes without WebAssembly, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while mobile storage remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, current mobile compatibility fixes, and earlier Open BSV grants.", - "migration": "Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/." + "summary": "Adds the built-in mobile BRC-177 noSend-expiry signer, remote storage capability negotiation, default-monitor ownership coordination across restarts and devices, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback for React Native runtimes without WebAssembly, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while mobile storage remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, current mobile compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use.", + "migration": "Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness." }, { "name": "create-bsv-app", diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index aad0934bd..c75e4504c 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -328,8 +328,8 @@ "@bsv/btms-permission-module": "1.1.4", "@bsv/ecpm-permission-module": "0.1.0", "@bsv/wallet-relay": "0.3.6", - "@bsv/wallet-toolbox-client": "2.11.0", - "@bsv/wallet-toolbox-mobile": "2.11.0", - "@bsv/wallet-toolbox": "2.11.0" + "@bsv/wallet-toolbox-client": "2.11.1", + "@bsv/wallet-toolbox-mobile": "2.11.1", + "@bsv/wallet-toolbox": "2.11.1" } } diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index 15c1b6e53..a2ff7a339 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -6,6 +6,17 @@ attention to changes that materially alter behavior or extend functionality. ## wallet-toolbox (unreleased) +- Secure final overlay identity discovery with independently verified transaction + evidence, byte-derived txid/hint checks, output bounds, the subject-signed + identity envelope, and explicit certificate success. Cached overlay responses + retain evidence only and repeat verification before returning identities. + Configure the existing wallet Services ChainTracker; direct `identityUtils` + callers must pass it explicitly. Local contacts, permission handling, wallet + RPC shapes, and current pagination behavior are retained. This security fix is + a patch in the full, browser, and mobile packages (2.11.1). No storage migration + is required. See [identity verification](docs/identity-verification.md) for the + supported token contract and remaining chain-cache/discovery limits. + - Keep Argon2id-backed UMP v3 wallets available in React Native and other runtimes without WebAssembly by falling back to an asynchronously yielding, standards-compatible JavaScript implementation. The same KDF parameters and diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index 75f6cfa69..e140f2c1b 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -424,6 +424,17 @@ does not apply schema changes. ## Development +### Overlay identity verification + +Final identity discovery verifies transaction evidence against the wallet's +existing `Services.getChainTracker()` configuration before validating the +subject-signed identity output and certificate. Cached responses are reverified +before use. Direct `identityUtils` callers must supply a canonical `ChainTracker`; +missing context or invalid evidence produces no overlay identities. Local +contacts retain their separate policy. Inclusion does not establish unspentness +or freshness. See [identity verification](docs/identity-verification.md) for +configuration, compatibility, and remaining verification lifecycle limits. + ```bash git clone https://github.com/bsv-blockchain/ts-stack.git cd ts-stack @@ -445,9 +456,12 @@ network access, or long runtimes. Files named `*.live.test.ts` are public-networ checks, also excluded from deterministic PR coverage. Run exactly one governed suite with `test:manual -- ` or `test:live -- ` after reviewing `governance/test-quality/policy.json`; never batch-run operator suites. CI -merges four Wallet Toolbox coverage shards -for reporting; the complete local `test:coverage` run currently measures -69.12% statements, 59.09% branches, 72.83% functions, and 71.06% lines. +merges four Wallet Toolbox coverage shards for reporting. The C01 local +`test:coverage --runInBand` run passed 225 suites and 2,188 tests, with one +pre-existing skipped test. Its all-files totals were 45.75% statements, 38.86% +branches, 42.57% functions, and 45.46% lines; that collection includes imported +`out/src` code as well as source files. Use the exact run's coverage report, +rather than comparing unlike source-only and combined collections. Operational repair, migration, export, and long-running service procedures are not tests. They live under [`operator/`](./operator/README.md), produce an exact diff --git a/packages/wallet/wallet-toolbox/client/README.md b/packages/wallet/wallet-toolbox/client/README.md index 295abbe26..de07e7257 100644 --- a/packages/wallet/wallet-toolbox/client/README.md +++ b/packages/wallet/wallet-toolbox/client/README.md @@ -126,6 +126,13 @@ CSP is an application and deployment concern rather than a package-level access ## Contributor checks +Final overlay identity discovery uses the configured wallet Services ChainTracker +to verify transaction evidence before returning certificate results, including +on response-cache hits. Local contacts retain their separate policy. See the +[identity verification guide](../docs/identity-verification.md) for configuration, +compatibility, and chain-cache limits; inclusion is not proof of freshness or +unspentness. + From the repository root, build the SDK and package before running the installed-consumer browser gate: ```bash diff --git a/packages/wallet/wallet-toolbox/client/package.json b/packages/wallet/wallet-toolbox/client/package.json index a004687ff..1e2e03a3f 100644 --- a/packages/wallet/wallet-toolbox/client/package.json +++ b/packages/wallet/wallet-toolbox/client/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-client", - "version": "2.11.0", + "version": "2.11.1", "type": "module", "sideEffects": false, "engines": { diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 921e99103..734933117 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,13 +2,13 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1693000, - "gzip": 400000, + "raw": 1696000, + "gzip": 401000, "brotli": 314000 }, "esbuild": { - "raw": 1321000, - "gzip": 364000, + "raw": 1324000, + "gzip": 365000, "brotli": 293000 } } diff --git a/packages/wallet/wallet-toolbox/docs/identity-verification.md b/packages/wallet/wallet-toolbox/docs/identity-verification.md new file mode 100644 index 000000000..40d036a81 --- /dev/null +++ b/packages/wallet/wallet-toolbox/docs/identity-verification.md @@ -0,0 +1,106 @@ +# Identity overlay verification + +This page records the bounded C01 source contract for identity results returned by the wallet toolbox. It is an inventory of the current implementation and its compatibility edges. It does not describe completion of the wider overlay reliability plan. + +## Evidence boundary + +An overlay lookup result is a host-supplied `LookupAnswer`. The resolver permits an output to carry a transaction id hint, but the hint is metadata rather than chain authority. C01 treats each output as untrusted evidence and verifies the BEEF bytes and selected output with the wallet's configured `ChainTracker` before decoding an identity certificate. + +The shared transaction seam is [`verifyOverlayOutput`](../src/utility/verifyOverlayOutput.ts). It owns a copy of the BEEF bytes, derives the transaction id from those bytes, and rejects a supplied hint unless it matches that derived id. It checks the selected output and verifies the transaction graph through the caller's tracker. An unconfirmed branch must have ancestry; an input-free unconfirmed leaf cannot anchor an identity result. The preflight also rejects duplicate inputs and conflicting spends across distinct unconfirmed ancestors, while allowing a shared transaction to be traversed once. This is consistency of the supplied graph, not an external unspentness check. The generic helper has no network default, certificate policy, verdict cache, or identity-specific locking-key rule. + +The wallet obtains the tracker through [`WalletServicesOptions.chainTracker`](../src/sdk/WalletServices.interfaces.ts) and [`Services.getChainTracker`](../src/services/Services.ts). A configured tracker is the wallet's chain authority. The utility functions [`queryOverlay`](../src/utility/identityUtils.ts), [`parseResults`](../src/utility/identityUtils.ts), and [`parseResults$`](../src/utility/identityUtils.ts) accept an explicit optional tracker; when it is missing they fail closed with an empty result. This keeps a caller from accidentally treating an overlay response as verified. + +The identity parser then decodes the selected output as PushDrop, requires the subject-signed field payload to verify, requires `VerifiableCertificate.verify()` to return `true`, decrypts the public keyring, and requires nonempty decrypted fields. A cryptographic signature failure is already surfaced as an exception by [`ProtoWallet.verifySignature`](../../../sdk/src/wallet/ProtoWallet.ts), which the parser drops with the candidate. The earlier ignored boolean from certificate verification is therefore not an invalid-certificate bypass; C01 makes the success condition explicit while adding the missing transaction evidence checks. Candidate errors are not logged because parser/decryption exceptions may contain identity data. + +## Standard identity envelope + +The interoperable envelope is the one emitted by the default [`IdentityClientOptions`](../../../sdk/src/identity/types/index.ts): protocol ID `[1, 'identity']`, key ID `'1'`, token amount `1`, and output index `0`. [`IdentityClient.publiclyRevealAttributes`](../../../sdk/src/identity/IdentityClient.ts) creates the PushDrop output and broadcasts it to `tm_identity`. + +The current [`IdentityTopicManager`](../../../../packages/overlays/topics/src/identity/IdentityTopicManager.ts) verifies the same subject-signed PushDrop fields with protocol `[1, 'identity']` and key ID `'1'`, then checks certificate validity and nonempty decrypted attributes. C01 matches that actual server contract. `IdentityClientOptions` exposes custom protocol and key values, but the current topic manager does not accept arbitrary values: a non-default protocol ID or key ID is a compatibility hazard and can cause the topic to reject the output. C01 does not widen topic acceptance. + +There is no existing topic contract for comparing an identity certificate to an arbitrary `lockingPublicKey`, so C01 adds no such equality rule. The subject-signed envelope, certificate signature, successful decryption, and trusted-certifier policy remain the relevant checks. + +## Cache and contact boundaries + +The wallet's overlay evidence cache is a two-minute response cache in [`Wallet.ts`](../src/Wallet.ts). Cached BEEF is revalidated on every use with the current tracker and certificate checks. If any candidate is rejected, the query entry is evicted so a later call can fetch fresh evidence. The cache is not a chain verdict and does not change local contact behavior. + +Local contacts are a separate wallet-owned source. [`Wallet.ts`](../src/Wallet.ts) synthesizes contact results with the local contact's subject/certifier relationship and local trust data. A contact hit may short-circuit the overlay path as before; it must retain local-contact provenance and must not be presented as an overlay SPV result. Contact lookup failures fall through to the network path. + +## Compatibility inventory and limits + +The public identity path spans the SDK wallet interfaces and clients, JSON and binary wallet transports, toolbox wallet managers, and [`IdentityClient`](../../../sdk/src/identity/IdentityClient.ts). C01 preserves those Promise method shapes and the `parseResults$` async-iterable shape. It does not change permission negotiation or pagination. The interface documentation says `seekPermission` defaults true, while the validator currently applies a false default; see [`Wallet.interfaces.ts`](../../../sdk/src/wallet/Wallet.interfaces.ts) and [`validationHelpers.ts`](../../../sdk/src/wallet/validationHelpers.ts). The wallet's current overlay calls also do not forward validated `limit` and `offset`, although the identity lookup service accepts them. These are compatibility characterization items for W00/W02, outside this C01 document. + +The resolver currently validates only the shape of a txid hint in [`LookupResolver.ts`](../../../sdk/src/overlay-tools/LookupResolver.ts). Its aggregation deduplicates by the hinted or derived txid and output index, keeping the first answer, and its fast path trusts a nonempty hint ([`LookupResolver.ts`](../../../sdk/src/overlay-tools/LookupResolver.ts), [`LookupResolver.ts`](../../../sdk/src/overlay-tools/LookupResolver.ts)). C02 owns the first-wins raw resolver suppression and pending full txid sharing work. C01 verifies the evidence that reaches the wallet; it cannot recover an alternate candidate discarded before parsing. + +The existing `ChaintracksChainTracker` still has its height-keyed root-cache and reorganization lifecycle. C01 makes no reorganization-safety claim; C03 covers that Chaintracks cache limitation and lifecycle work. C01 adds no workers, shared transaction jobs, response/graph budgets, or whole-attempt deadline. Large proofs still incur parsing and verification work on the calling runtime. These limits and canonical-context fencing require the later coordinator/runtime slices; no latency, reorganization safety, or deployment claim is made here. + +## Requirement and verification mapping + +The scoped source work supports requirement V1 (independent transaction evidence), the byte-binding portion of V2 (BEEF-derived txid must match any hint), and V5 (certificate success and trust policy). The planned characterization cases are T04 (false txid hint), T07 (confirmed/unconfirmed ancestry, scripts, values, and graph-internal conflicts), T09 (output/envelope/certificate validity), T10 (permission, cache, and local contacts), and T13 (Promise and pagination compatibility). This mapping is evidence for the C01 slice only; it is not a claim that the full verification plan has passed. + +## Configuration and migration + +This is a patch security correction in the 2.11.1 full, browser, and mobile +packages; the aggregate release-note candidate remains minor relative to the +recorded 2.10.4 published baseline. Wallet RPC and stored data need no migration. +Wallet builders keep their existing `Services` chain configuration. Its +`getChainTracker()` selects `options.chainTracker`, or wraps the configured +`options.chaintracks`; a configuration/availability error never becomes an +acceptance verdict. Use a chain source maintained independently of overlay hosts. + +Direct utility callers previously supplied only an answer. They must now pass +their canonical chain source: + +```ts +const tracker = await wallet.getServices().getChainTracker() +const certificates = await parseResults(answer, tracker) +``` + +Omitting the optional argument remains source-compatible but fails closed. +There is no bypass toggle. Failed candidate evidence evicts a wallet response +cache entry; failures never establish a permanent negative verdict for a txid. +Untrusted cached bytes may be shared within this wallet, but decrypted result +objects are rebuilt on each call and then passed through the existing trust +settings. The existing two-minute trust-settings snapshot policy is unchanged. + +## Package size review + +On 8 September 2026, the originating review task +`01a081b5-26d4-7ad1-8d85-243fe238d595` explicitly approved these measured C01 +budget adjustments under +[`governance/browser-artifact-policy.json`](../../../../governance/browser-artifact-policy.json). +The policy requires a versioned source change, composition evidence, and explicit +review. All three published artifacts advance from 2.11.0 to 2.11.1. Mandatory +transaction, graph, and identity checks remain in the portable bundles. + +Measurements used Node 24.15.0 and pnpm 10.33.2 on the same macOS host, with +base commit `2bc799a8d8e535242e6de2d305f426ce3975ea7b` extracted into a temporary +source tree and built against the same unchanged SDK and dependency graph. +`pnpm build` ran in each base/current client and mobile package. A temporary +copy of the platform checker printed every size instead of evaluating budgets: +`node /tmp/c01-measure-baseline.mjs browser`, +`node /tmp/c01-measure-baseline.mjs mobile`, +`node /tmp/c01-measure-platform.mjs browser`, and +`node /tmp/c01-measure-platform.mjs mobile`. +These were **measurements only, not passing platform gates**. The original +platform checker was not modified. + +Each cell lists raw / gzip / Brotli bytes: + +| Consumer | Base | C01 | Reviewed maximum | +| -------- | --------------------------------- | --------------------------------- | --------------------------------- | +| Vite | 1,692,309 / 399,380 / 312,207 | 1,694,805 / 400,062 / 312,426 | 1,696,000 / 401,000 / 314,000 | +| esbuild | 1,320,184 / 363,792 / 291,500 | 1,322,211 / 364,400 / 291,926 | 1,324,000 / 365,000 / 293,000 | +| Metro | 1,747,262 / 443,100 / 343,780 | 1,749,640 / 443,811 / 343,927 | 1,751,000 / 455,000 / 360,000 | +| Hermes | 3,544,570 / 1,440,174 / 1,117,759 | 3,550,004 / 1,442,134 / 1,120,813 | 3,553,000 / 1,443,000 / 1,123,000 | + +Raw growth is 2,496 / 2,027 / 2,378 / 5,434 bytes respectively (about +0.14–0.15%). Vite composition retains 106 modules and the same packages: +`@bsv/sdk`, `@bsv/wallet-toolbox-client`, `@noble/hashes`, `hash-wasm`, and `idb`. +The esbuild module count remains 173. No new dependency or platform-only import +was added. Shared wallet cache/fetch logic removes duplication; independent +verification and subject-envelope checks account for the added code. Only +exceeded dimensions changed, rounded to preserve comparable existing margins; +all other limits stay fixed. This is a reviewed security-feature payload change, +not an analysis exception. Original `test:browser` and `test:mobile` commands +remain the executable gates and are recorded separately in the delivery ledger. diff --git a/packages/wallet/wallet-toolbox/mobile/README.md b/packages/wallet/wallet-toolbox/mobile/README.md index 9af9ad398..3dab43aa3 100644 --- a/packages/wallet/wallet-toolbox/mobile/README.md +++ b/packages/wallet/wallet-toolbox/mobile/README.md @@ -85,6 +85,13 @@ Capability negotiation fails before prefunding against an older server. See ## Use cases +Final overlay identity discovery uses the configured wallet Services ChainTracker +to verify transaction evidence before returning certificate results, including +on response-cache hits. Local contacts retain their separate policy. See the +[identity verification guide](../docs/identity-verification.md) for configuration, +compatibility, and chain-cache limits; inclusion is not proof of freshness or +unspentness. + ### Self-custody BSV wallet on a phone Run the BRC-100 wallet entirely on-device, with remote storage for cross-device sync. diff --git a/packages/wallet/wallet-toolbox/mobile/package.json b/packages/wallet/wallet-toolbox/mobile/package.json index 6e7e72cc8..716215b97 100644 --- a/packages/wallet/wallet-toolbox/mobile/package.json +++ b/packages/wallet/wallet-toolbox/mobile/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-mobile", - "version": "2.11.0", + "version": "2.11.1", "type": "module", "sideEffects": false, "engines": { diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 53027c0aa..69ddeddf0 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -2,13 +2,13 @@ "profile": "mobile", "maximumBytes": { "metro": { - "raw": 1748000, + "raw": 1751000, "gzip": 455000, "brotli": 360000 }, "hermes": { - "raw": 3547000, - "gzip": 1441000, + "raw": 3553000, + "gzip": 1443000, "brotli": 1123000 } } diff --git a/packages/wallet/wallet-toolbox/package.json b/packages/wallet/wallet-toolbox/package.json index 82c024a7b..0ee32633f 100644 --- a/packages/wallet/wallet-toolbox/package.json +++ b/packages/wallet/wallet-toolbox/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox", - "version": "2.11.0", + "version": "2.11.1", "sideEffects": false, "type": "commonjs", "engines": { diff --git a/packages/wallet/wallet-toolbox/src/Wallet.ts b/packages/wallet/wallet-toolbox/src/Wallet.ts index e5ffb55ac..65a8cedaf 100644 --- a/packages/wallet/wallet-toolbox/src/Wallet.ts +++ b/packages/wallet/wallet-toolbox/src/Wallet.ts @@ -63,6 +63,8 @@ import { MasterCertificate, Certificate, LookupResolver, + LookupAnswer, + VerifiableCertificate, AtomicBEEF, BEEF, KeyDeriverApi, @@ -79,7 +81,7 @@ import { createAction, CreateActionResultX } from './signer/methods/createAction import { signAction, SignActionResultX } from './signer/methods/signAction' import { internalizeAction } from './signer/methods/internalizeAction' import { WalletSettingsManager } from './WalletSettingsManager' -import { queryOverlay, transformVerifiableCertificatesWithTrust } from './utility/identityUtils' +import { parseResults, queryOverlayEvidence, transformVerifiableCertificatesWithTrust } from './utility/identityUtils' import { maxPossibleSatoshis } from './storage/methods/generateChange' import { hasBrc177NoSendExpiryLabel, parseBrc177NoSendExpiryLabels } from './utility/brc177NoSendExpiry' import { createNoSendExpiryAction } from './signer/methods/createNoSendExpiryAction' @@ -791,8 +793,32 @@ export class Wallet implements WalletInterface, ProtoWallet { trustSettings: Awaited>['trustSettings'] } - /** 2-minute cache of queryOverlay() results keyed by normalized query */ - private readonly _overlayCache: Map = new Map() + /** Two-minute untrusted response cache. Every use repeats transaction and identity checks. */ + private readonly _overlayEvidenceCache = new Map() + + private async discoverOverlayCertificates( + query: unknown, + cacheKey: string, + forceRefresh: boolean, + now: number + ): Promise { + // Use the wallet's existing network/chain configuration, never the overlay host's verdict. + const chainTracker = await this.getServices().getChainTracker() + let cached = forceRefresh ? undefined : this._overlayEvidenceCache.get(cacheKey) + if (cached == null || cached.expiresAt <= now) { + const value = await queryOverlayEvidence(query, this.lookupResolver) + cached = { value, expiresAt: now + 2 * 60 * 1000 } + this._overlayEvidenceCache.set(cacheKey, cached) + } + if (cached.value.type !== 'output-list') { + this._overlayEvidenceCache.delete(cacheKey) + return [] + } + const certificates = await parseResults(cached.value, chainTracker) + // Failed evidence must allow another fetch, including after temporary chain unavailability. + if (certificates.length !== cached.value.outputs.length) this._overlayEvidenceCache.delete(cacheKey) + return certificates + } async discoverByIdentityKey( args: DiscoverByIdentityKeyArgs & { forceRefresh?: boolean }, @@ -831,25 +857,20 @@ export class Wallet implements WalletInterface, ProtoWallet { const certifiers = trustSettings.trustedCertifiers.map(c => c.identityKey).sort((a, b) => a.localeCompare(b)) - // --- queryOverlay cache (2 minutes, client-side, bounded staleness) --- + // --- Untrusted overlay response cache; verify again before use. --- const cacheKey = JSON.stringify({ fn: 'discoverByIdentityKey', identityKey: args.identityKey, certifiers }) - let cached = forceRefresh ? undefined : this._overlayCache.get(cacheKey) - if (cached == null || cached.expiresAt <= now) { - const value = await queryOverlay({ identityKey: args.identityKey, certifiers }, this.lookupResolver) - cached = { value, expiresAt: now + TTL_MS } - this._overlayCache.set(cacheKey, cached) - } - - if (!cached.value) { - return { totalCertificates: 0, certificates: [] } - } - - return transformVerifiableCertificatesWithTrust(trustSettings, cached.value as any) + const certificates = await this.discoverOverlayCertificates( + { identityKey: args.identityKey, certifiers }, + cacheKey, + forceRefresh, + now + ) + return transformVerifiableCertificatesWithTrust(trustSettings, certificates) } async discoverByAttributes( @@ -897,25 +918,20 @@ export class Wallet implements WalletInterface, ProtoWallet { attributesKey = JSON.stringify(args.attributes, keys) } - // --- queryOverlay cache (2 minutes, client-side, bounded staleness) --- + // --- Untrusted overlay response cache; verify again before use. --- const cacheKey = JSON.stringify({ fn: 'discoverByAttributes', attributes: attributesKey, certifiers }) - let cached = forceRefresh ? undefined : this._overlayCache.get(cacheKey) - if (cached == null || cached.expiresAt <= now) { - const value = await queryOverlay({ attributes: args.attributes, certifiers }, this.lookupResolver) - cached = { value, expiresAt: now + TTL_MS } - this._overlayCache.set(cacheKey, cached) - } - - if (!cached.value) { - return { totalCertificates: 0, certificates: [] } - } - - return transformVerifiableCertificatesWithTrust(trustSettings, cached.value as any) + const certificates = await this.discoverOverlayCertificates( + { attributes: args.attributes, certifiers }, + cacheKey, + forceRefresh, + now + ) + return transformVerifiableCertificatesWithTrust(trustSettings, certificates) } verifyReturnedTxidOnly(beef: Beef, knownTxids?: string[]): Beef { diff --git a/packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts b/packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts new file mode 100644 index 000000000..23ce78fcc --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts @@ -0,0 +1,176 @@ +import { KeyDeriver, LookupAnswer, LookupResolver, PrivateKey, Validation } from '@bsv/sdk' +import { Wallet } from '../Wallet' +import { WalletSettingsManager } from '../WalletSettingsManager' +import { WalletStorageManager } from '../storage/WalletStorageManager' +import { WalletServices } from '../sdk/WalletServices.interfaces' +import { + createIdentityVerificationFixture, + IdentityVerificationFixture +} from '../utility/__tests__/identityVerification.fixtures' + +function walletFor(fixture: IdentityVerificationFixture) { + const keyDeriver = new KeyDeriver(new PrivateKey(15)) + const trustSettings = { + trustLevel: 1, + trustedCertifiers: [ + { identityKey: fixture.certificate.certifier, name: 'Synthetic certifier', description: '', trust: 1 } + ] + } + const query = jest.fn(async (): Promise => ({ + type: 'output-list', + outputs: [{ beef: fixture.certificateBEEF, outputIndex: 0 }] + })) + const getChainTracker = jest.fn(async () => fixture.confirmedTracker) + const getSettings = jest.fn(async () => ({ trustSettings })) + const wallet = new Wallet({ + chain: 'main', + keyDeriver, + storage: new WalletStorageManager(keyDeriver.identityKey), + services: { getChainTracker } as unknown as WalletServices, + lookupResolver: { query } as unknown as LookupResolver, + settingsManager: { get: getSettings } as unknown as WalletSettingsManager + }) + return { wallet, query, getChainTracker, getSettings, trustSettings } +} + +describe('Wallet final identity verification and compatibility', () => { + let fixture: IdentityVerificationFixture + beforeEach(async () => { + fixture = await createIdentityVerificationFixture() + }) + + it.each(['identity', 'attributes'] as const)('verifies evidence again on a cached %s discovery', async method => { + const { wallet, query, getChainTracker } = walletFor(fixture) + const discover = () => + method === 'identity' + ? wallet.discoverByIdentityKey({ identityKey: fixture.certificate.subject }, 'app.example') + : wallet.discoverByAttributes({ attributes: { name: 'Alice' } }, 'app.example') + + await expect(discover()).resolves.toMatchObject({ totalCertificates: 1 }) + const checked = fixture.confirmedTracker.checkedRoots.length + await expect(discover()).resolves.toMatchObject({ totalCertificates: 1 }) + expect(query).toHaveBeenCalledTimes(1) + expect(getChainTracker).toHaveBeenCalledTimes(2) + expect(fixture.confirmedTracker.checkedRoots.length).toBeGreaterThan(checked) + + // The configured source changes its verdict. This tests the wallet cache, not + // reorg detection/invalidation inside a particular ChainTracker implementation. + const roots = [...fixture.confirmedTracker.roots] + fixture.confirmedTracker.roots.clear() + await expect(discover()).resolves.toEqual({ totalCertificates: 0, certificates: [] }) + roots.forEach(root => fixture.confirmedTracker.roots.add(root)) + await expect(discover()).resolves.toMatchObject({ totalCertificates: 1 }) + expect(query).toHaveBeenCalledTimes(2) + }) + + it('does not let host or returned-certificate mutation change cached evidence', async () => { + const { wallet, query } = walletFor(fixture) + const args = { identityKey: fixture.certificate.subject } + const first = await wallet.discoverByIdentityKey(args) + fixture.certificateBEEF.fill(0) + first.certificates[0].decryptedFields.name = 'Changed locally' + const second = await wallet.discoverByIdentityKey(args) + expect(second.certificates[0].decryptedFields).toEqual({ name: 'Alice' }) + expect(query).toHaveBeenCalledTimes(1) + }) + + it('keeps trust filtering and forceRefresh on the final Promise API', async () => { + const { wallet, query, trustSettings } = walletFor(fixture) + const args = { identityKey: fixture.certificate.subject } + const promise = wallet.discoverByIdentityKey(args) + expect(promise).toBeInstanceOf(Promise) + await expect(promise).resolves.toMatchObject({ totalCertificates: 1 }) + trustSettings.trustLevel = 2 + await expect(wallet.discoverByIdentityKey(args)).resolves.toEqual({ totalCertificates: 0, certificates: [] }) + trustSettings.trustLevel = 1 + await expect(wallet.discoverByIdentityKey({ ...args, forceRefresh: true })).resolves.toMatchObject({ + totalCertificates: 1 + }) + expect(query).toHaveBeenCalledTimes(2) + trustSettings.trustedCertifiers = [] + await expect(wallet.discoverByIdentityKey(args)).resolves.toEqual({ totalCertificates: 0, certificates: [] }) + }) + + it('preserves local contact provenance and bypasses chain/network work for contacts', async () => { + const { wallet, query, getChainTracker, getSettings } = walletFor(fixture) + const contact = { identityKey: fixture.certificate.subject, decryptedFields: { name: 'Local Alice' } } + wallet.contactSource = { + findByIdentityKey: async () => contact, + findByAttributes: async () => [contact] + } + wallet.services = undefined + for (const result of [ + await wallet.discoverByIdentityKey({ identityKey: contact.identityKey }), + await wallet.discoverByAttributes({ attributes: { name: 'Local Alice' } }) + ]) { + expect(result).toMatchObject({ + totalCertificates: 1, + certificates: [ + { + type: 'contact', + signature: '', + decryptedFields: { name: 'Local Alice' }, + certifierInfo: { trust: Infinity } + } + ] + }) + } + expect(query).not.toHaveBeenCalled() + expect(getChainTracker).not.toHaveBeenCalled() + expect(getSettings).not.toHaveBeenCalled() + await expect( + wallet.discoverByIdentityKey({ identityKey: contact.identityKey, forceRefresh: true }) + ).rejects.toThrow() + }) + + it('rejects missing chain configuration before overlay lookup', async () => { + const { wallet, query, getChainTracker } = walletFor(fixture) + getChainTracker.mockRejectedValueOnce(new Error('Canonical source unavailable')) + await expect(wallet.discoverByIdentityKey({ identityKey: fixture.certificate.subject })).rejects.toThrow( + 'Canonical source unavailable' + ) + expect(query).not.toHaveBeenCalled() + }) + + it('fails closed for a non-output-list response without caching it or throwing a TypeError', async () => { + const { wallet, query } = walletFor(fixture) + query.mockResolvedValueOnce({ type: 'freeform', result: { untrusted: true } } as unknown as LookupAnswer) + const args = { identityKey: fixture.certificate.subject } + await expect(wallet.discoverByIdentityKey(args)).resolves.toEqual({ totalCertificates: 0, certificates: [] }) + await expect(wallet.discoverByIdentityKey(args)).resolves.toMatchObject({ totalCertificates: 1 }) + expect(query).toHaveBeenCalledTimes(2) + }) + + it('characterizes legacy pagination and seekPermission without changing forwarding', async () => { + const { wallet, query } = walletFor(fixture) + const identityArgs = { identityKey: fixture.certificate.subject, limit: 1, offset: 3 } + expect(Validation.validateDiscoverByIdentityKeyArgs({ identityKey: fixture.certificate.subject })).toMatchObject({ + limit: 10, + offset: 0, + seekPermission: false + }) + expect(Validation.validateDiscoverByAttributesArgs({ attributes: { name: 'Alice' } })).toMatchObject({ + limit: 10, + offset: 0, + seekPermission: false + }) + await expect(wallet.discoverByIdentityKey(identityArgs)).resolves.toMatchObject({ totalCertificates: 1 }) + expect(query).toHaveBeenLastCalledWith( + { + service: 'ls_identity', + query: { identityKey: fixture.certificate.subject, certifiers: [fixture.certificate.certifier] } + }, + undefined, + { graceMs: 300 } + ) + await wallet.discoverByAttributes({ attributes: { name: 'Alice' }, limit: 1, offset: 3, seekPermission: true }) + expect(query).toHaveBeenLastCalledWith( + { service: 'ls_identity', query: { attributes: { name: 'Alice' }, certifiers: [fixture.certificate.certifier] } }, + undefined, + { graceMs: 300 } + ) + await expect(wallet.discoverByIdentityKey({ ...identityArgs, limit: 10_001 })).rejects.toThrow() + await expect(wallet.discoverByAttributes({ attributes: {}, offset: -1 })).rejects.toThrow() + expect(query).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/utility/__tests__/fixtures/identity-verification.json b/packages/wallet/wallet-toolbox/src/utility/__tests__/fixtures/identity-verification.json new file mode 100644 index 000000000..62e73fb1c --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/utility/__tests__/fixtures/identity-verification.json @@ -0,0 +1,4 @@ +{ + "certificateBEEF": "AQC+7wH+YK4KAAECAABCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQgECl2dy9ADFnVCeuMA5R3KlEhfmb4j4Pqoc8xPSELmlbC0BAQAAAAABCQAAAAAAAAD9WgMhA4aNtbOPYqHd8rvAci59Wcv2cpkCpYJfqMBEHMPQFfbGrE3rAnsidHlwZSI6IkFRRUJBUUVCQVFFQkFRRUJBUUVCQVFFQkFRRUJBUUVCQVFFQkFRRUJBUUU9Iiwic2VyaWFsTnVtYmVyIjoiQWdJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnST0iLCJzdWJqZWN0IjoiMDM3NzRhZTdmODU4YTk0MTFlNWVmNDI0NmI3MGM2NWFhYzU2NDk5ODBiZTVjMTc4OTFiYmVjMTc4OTVkYTAwOGNiIiwiY2VydGlmaWVyIjoiMDNkMDExMTVkNTQ4ZTc1NjFiMTVjMzhmMDA0ZDczNDYzMzY4N2NmNDQxOTYyMDA5NWJjNWIwZjQ3MDcwYWZlODVhIiwicmV2b2NhdGlvbk91dHBvaW50IjoiMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMS4wIiwiZmllbGRzIjp7Im5hbWUiOiJQUDlBNlRRU3NrSXZFcHA0cDhQamQ2MHRYUms2ekNvMkdSV0ErRGZja0lmejE2c0tpWENYNHR2bGFLM3hJQUxTZmRJbUlOVT0ifSwic2lnbmF0dXJlIjoiMzA0NTAyMjEwMGRjOWZiYjg3MWViOGVkZGU2YTFlZDJjMGRkNWNhZWRhOGQzOTE0ZjI5YzliNTczYTUzMWQ5NDNhYzFkMjY0OWUwMjIwMGJiZTMzNjc3NzYxNDBjNjY0YTQzNDRlNDk2ZGFkNGE4MmQ2N2QxZGNjN2ZlZDg5OTJkMzhiYjJjZTIyZTAwMSIsImtleXJpbmciOnsibmFtZSI6ImZIMklUTlFoYnIvSkdobUs2QVBEUThVRkc5ZXowMHcrempEWmY1bUhaVitNb1NpL0pXaEpzWEJTeWFaMU4yUWxVMUZHY2FxVzRNY25wNmNrZ3pzcjk3SGgzeElTSHlrb09qaVJWdVZSaE1zPSJ9fUcwRQIhAL3vVAviqbcyhtgqNiXHxK33GyRwtfkU5Bgy/470pdPlAiBmzWHNcSJqUgfPQdH/ZuxmujeXrCBU7wxnnOCAn9zlGW0AAAAAAQA=", + "certificateBEEFSha256": "86df273e253fd47750ad25797712c943d628eecf6db115a2b6649366aa274d81" +} diff --git a/packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.fixtures.ts b/packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.fixtures.ts new file mode 100644 index 000000000..92e567b36 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.fixtures.ts @@ -0,0 +1,149 @@ +import { + ChainTracker, + MerklePath, + P2PKH, + PrivateKey, + ProtoWallet, + PushDrop, + Script, + Transaction, + Utils, + VerifiableCertificate +} from '@bsv/sdk' +import { createHash } from 'node:crypto' +import fixtureJson from './fixtures/identity-verification.json' + +// Synthetic fixture: fixed subject/certifier keys 11/12; pinned BEEF SHA-256 is checked before use. + +export const IDENTITY_VERIFICATION_PROTOCOL: [1, string] = [1, 'identity'] +export const IDENTITY_VERIFICATION_KEY_ID = '1' +export const IDENTITY_VERIFICATION_CONFIRMED_HEIGHT = 700_000 + +/** A local canonical-root source used by identity verification fixtures. */ +export class IdentityVerificationChainTracker implements ChainTracker { + readonly roots = new Set() + readonly checkedRoots: Array<{ root: string; height: number }> = [] + + async currentHeight(): Promise { + return IDENTITY_VERIFICATION_CONFIRMED_HEIGHT + 101 + } + + async isValidRootForHeight(root: string, height: number): Promise { + this.checkedRoots.push({ root, height }) + return height === IDENTITY_VERIFICATION_CONFIRMED_HEIGHT && this.roots.has(root) + } +} + +export interface IdentityVerificationFixture { + certificate: VerifiableCertificate + subjectWallet: ProtoWallet + certificateTransaction: Transaction + certificateBEEF: number[] + confirmedTracker: IdentityVerificationChainTracker + unconfirmedTransaction: Transaction + unconfirmedBEEF: number[] +} + +function sha256d(bytes: Uint8Array): Uint8Array { + return createHash('sha256').update(createHash('sha256').update(bytes).digest()).digest() +} + +/** Independent Node SHA-256d reference for a serialized transaction ID. */ +export function fixtureTxidFromBytes(bytes: Uint8Array): string { + return Buffer.from(sha256d(bytes)).reverse().toString('hex') +} + +/** Independent Bitcoin Merkle reference over display-order transaction IDs. */ +export function fixtureMerkleRoot(left: string, right: string): string { + const leftLE = Buffer.from(left, 'hex').reverse() + const rightLE = Buffer.from(right, 'hex').reverse() + return Buffer.from(sha256d(Buffer.concat([leftLE, rightLE]))) + .reverse() + .toString('hex') +} + +/** Attach a two-leaf, non-coinbase proof and register its root in the local tracker. */ +export function confirmIdentityFixtureTransaction(tx: Transaction, tracker: IdentityVerificationChainTracker): void { + const txid = tx.id('hex') + if (fixtureTxidFromBytes(tx.toUint8Array()) !== txid) throw new Error('SDK transaction ID disagrees with SHA-256d') + const path = new MerklePath(IDENTITY_VERIFICATION_CONFIRMED_HEIGHT, [ + [ + { offset: 0, hash: '42'.repeat(32) }, + { offset: 1, hash: txid, txid: true } + ] + ]) + tx.merklePath = path + const root = fixtureMerkleRoot('42'.repeat(32), txid) + if (path.computeRoot(txid) !== root) throw new Error('SDK Merkle root disagrees with SHA-256d') + tracker.roots.add(root) +} + +/** + * Builds a local, signed identity certificate in a subject-signed PushDrop output. + * The returned unconfirmed transaction spends a separately confirmed P2PKH ancestor. + */ +export async function createIdentityVerificationFixture(): Promise { + const subjectWallet = new ProtoWallet(new PrivateKey(11)) + const certificateBEEF = Utils.toArray(fixtureJson.certificateBEEF, 'base64') + if (createHash('sha256').update(Buffer.from(certificateBEEF)).digest('hex') !== fixtureJson.certificateBEEFSha256) { + throw new Error('Identity fixture BEEF digest mismatch') + } + const certificateTransaction = Transaction.fromBEEF(certificateBEEF) + const decoded = PushDrop.decode(certificateTransaction.outputs[0].lockingScript) + const parsed = JSON.parse(Utils.toUTF8(decoded.fields[0])) + const certificate = new VerifiableCertificate( + parsed.type, + parsed.serialNumber, + parsed.subject, + parsed.certifier, + parsed.revocationOutpoint, + parsed.fields, + parsed.keyring, + parsed.signature + ) + const confirmedTracker = new IdentityVerificationChainTracker() + const certificateTxid = fixtureTxidFromBytes(certificateTransaction.toUint8Array()) + const certificateRoot = fixtureMerkleRoot('42'.repeat(32), certificateTxid) + if ( + certificateTransaction.id('hex') !== certificateTxid || + certificateTransaction.merklePath?.computeRoot(certificateTxid) !== certificateRoot + ) { + throw new Error('Pinned identity fixture does not match its SHA-256d references') + } + confirmedTracker.roots.add(certificateRoot) + + const spendingKey = new PrivateKey(13) + const ancestor = new Transaction() + ancestor.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + ancestor.addOutput({ satoshis: 10, lockingScript: new P2PKH().lock(spendingKey.toAddress()) }) + confirmIdentityFixtureTransaction(ancestor, confirmedTracker) + + const createCertificateSpend = async (): Promise => { + const transaction = new Transaction() + transaction.addInput({ + sourceTransaction: ancestor, + sourceOutputIndex: 0, + unlockingScriptTemplate: new P2PKH().unlock(spendingKey) + }) + transaction.addOutput({ satoshis: 9, lockingScript: certificateTransaction.outputs[0].lockingScript }) + await transaction.sign() + return transaction + } + const confirmedCertificateTransaction = await createCertificateSpend() + confirmIdentityFixtureTransaction(confirmedCertificateTransaction, confirmedTracker) + const unconfirmedTransaction = await createCertificateSpend() + + return { + certificate, + subjectWallet, + certificateTransaction: confirmedCertificateTransaction, + certificateBEEF: confirmedCertificateTransaction.toBEEF(), + confirmedTracker, + unconfirmedTransaction, + unconfirmedBEEF: unconfirmedTransaction.toBEEF() + } +} diff --git a/packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.test.ts b/packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.test.ts new file mode 100644 index 000000000..0ab698888 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.test.ts @@ -0,0 +1,396 @@ +import { + MerklePath, + P2PKH, + PrivateKey, + ProtoWallet, + PushDrop, + Script, + Transaction, + Utils, + VerifiableCertificate +} from '@bsv/sdk' +import { LookupAnswer, LookupResolver } from '@bsv/sdk' +import { parseResults, parseResults$, queryOverlay } from '../identityUtils' +import { verifyOverlayOutput } from '../verifyOverlayOutput' +import { + createIdentityVerificationFixture, + confirmIdentityFixtureTransaction, + IDENTITY_VERIFICATION_CONFIRMED_HEIGHT, + IDENTITY_VERIFICATION_KEY_ID, + IDENTITY_VERIFICATION_PROTOCOL, + IdentityVerificationChainTracker, + IdentityVerificationFixture +} from './identityVerification.fixtures' + +function answer(...outputs: LookupAnswer['outputs']): LookupAnswer { + return { type: 'output-list', outputs } +} + +function outputFor(beef: number[], outputIndex = 0, txid?: string): LookupAnswer['outputs'][number] { + return txid === undefined ? { beef, outputIndex } : { beef, outputIndex, txid } +} + +async function collect(iterable: AsyncIterable): Promise { + const values: VerifiableCertificate[] = [] + for await (const value of iterable) values.push(value) + return values +} + +describe('identity overlay verification', () => { + let fixture: IdentityVerificationFixture + + beforeEach(async () => { + fixture = await createIdentityVerificationFixture() + }) + + it('accepts a signed, decryptable certificate from a locally confirmed transaction', async () => { + await expect(fixture.certificateTransaction.verify(fixture.confirmedTracker)).resolves.toBe(true) + + const results = await parseResults( + answer(outputFor(fixture.certificateBEEF, 0, fixture.certificateTransaction.id('hex'))), + fixture.confirmedTracker + ) + + expect(results).toHaveLength(1) + expect(results[0].decryptedFields).toEqual({ name: 'Alice' }) + expect(fixture.confirmedTracker.checkedRoots).toContainEqual({ + root: fixture.certificateTransaction.merklePath?.computeRoot(fixture.certificateTransaction.id('hex')), + height: IDENTITY_VERIFICATION_CONFIRMED_HEIGHT + }) + }) + + it('accepts a genuinely signed unconfirmed spend only when its ancestor is confirmed', async () => { + await expect(fixture.unconfirmedTransaction.verify(fixture.confirmedTracker)).resolves.toBe(true) + + await expect( + parseResults(answer(outputFor(fixture.unconfirmedBEEF)), fixture.confirmedTracker) + ).resolves.toHaveLength(1) + }) + + it('rejects confirmed candidates with an invalid or unavailable chain root', async () => { + const noRoots = new IdentityVerificationChainTracker() + + await expect(parseResults(answer(outputFor(fixture.certificateBEEF)), noRoots)).resolves.toEqual([]) + }) + + it('rejects an unconfirmed candidate whose BEEF omits its ancestry', async () => { + const disconnected = Transaction.fromBEEF(fixture.unconfirmedBEEF) + disconnected.inputs[0].sourceTransaction = undefined + disconnected.inputs[0].sourceTXID = fixture.unconfirmedTransaction.inputs[0].sourceTransaction?.id('hex') as string + + await expect(parseResults(answer(outputFor(disconnected.toBEEF(true))), fixture.confirmedTracker)).resolves.toEqual( + [] + ) + }) + + it('rejects invalid spend scripts and values instead of accepting scripts-only evidence', async () => { + const invalidScript = Transaction.fromBEEF(fixture.unconfirmedBEEF) + invalidScript.inputs[0].unlockingScript = Script.fromASM('OP_FALSE') + const invalidValue = Transaction.fromBEEF(fixture.unconfirmedBEEF) + invalidValue.outputs[0].satoshis = 11 + + await expect(parseResults(answer(outputFor(invalidScript.toBEEF())), fixture.confirmedTracker)).resolves.toEqual([]) + await expect(parseResults(answer(outputFor(invalidValue.toBEEF())), fixture.confirmedTracker)).resolves.toEqual([]) + }) + + it.each([ + ['mismatched', '00'.repeat(32)], + ['empty', ''], + ['malformed', 'not-a-txid'] + ])('rejects a %s txid hint', async (_kind, txid) => { + await expect( + parseResults(answer(outputFor(fixture.certificateBEEF, 0, txid)), fixture.confirmedTracker) + ).resolves.toEqual([]) + }) + + it('rejects a validly signed transaction that spends the same confirmed outpoint twice', async () => { + const ancestor = fixture.unconfirmedTransaction.inputs[0].sourceTransaction! + const spendingKey = new PrivateKey(13) + const duplicateSpend = new Transaction() + for (let index = 0; index < 2; index++) { + duplicateSpend.addInput({ + sourceTransaction: ancestor, + sourceOutputIndex: 0, + unlockingScriptTemplate: new P2PKH().unlock(spendingKey) + }) + } + duplicateSpend.addOutput({ satoshis: 19, lockingScript: fixture.certificateTransaction.outputs[0].lockingScript }) + await duplicateSpend.sign() + + // SDK graph/value/script verification alone currently accepts duplicate outpoints. + await expect(duplicateSpend.verify(fixture.confirmedTracker)).resolves.toBe(true) + await expect(parseResults(answer(outputFor(duplicateSpend.toBEEF())), fixture.confirmedTracker)).resolves.toEqual( + [] + ) + }) + + it('rejects a joined graph whose distinct unconfirmed parents double-spend one confirmed outpoint', async () => { + const ancestor = fixture.unconfirmedTransaction.inputs[0].sourceTransaction! + const spendingKey = new PrivateKey(13) + const createParent = async (satoshis: number): Promise => { + const tx = new Transaction() + tx.addInput({ + sourceTransaction: ancestor, + sourceOutputIndex: 0, + unlockingScriptTemplate: new P2PKH().unlock(spendingKey) + }) + tx.addOutput({ satoshis, lockingScript: new P2PKH().lock(spendingKey.toAddress()) }) + await tx.sign() + return tx + } + const first = await createParent(9) + const second = await createParent(8) + expect(first.id('hex')).not.toEqual(second.id('hex')) + const joined = new Transaction() + for (const sourceTransaction of [first, second]) { + joined.addInput({ + sourceTransaction, + sourceOutputIndex: 0, + unlockingScriptTemplate: new P2PKH().unlock(spendingKey) + }) + } + joined.addOutput({ satoshis: 16, lockingScript: fixture.certificateTransaction.outputs[0].lockingScript }) + await joined.sign() + + await expect(joined.verify(fixture.confirmedTracker)).resolves.toBe(true) + await expect(parseResults(answer(outputFor(joined.toBEEF())), fixture.confirmedTracker)).resolves.toEqual([]) + }) + + it('rejects an identity child of an unconfirmed ancestor with duplicate inputs', async () => { + const ancestor = fixture.unconfirmedTransaction.inputs[0].sourceTransaction! + const spendingKey = new PrivateKey(13) + const duplicateAncestor = new Transaction() + for (let index = 0; index < 2; index++) { + duplicateAncestor.addInput({ + sourceTransaction: ancestor, + sourceOutputIndex: 0, + unlockingScriptTemplate: new P2PKH().unlock(spendingKey) + }) + } + duplicateAncestor.addOutput({ satoshis: 19, lockingScript: new P2PKH().lock(spendingKey.toAddress()) }) + await duplicateAncestor.sign() + const identityChild = new Transaction() + identityChild.addInput({ + sourceTransaction: duplicateAncestor, + sourceOutputIndex: 0, + unlockingScriptTemplate: new P2PKH().unlock(spendingKey) + }) + identityChild.addOutput({ satoshis: 9, lockingScript: fixture.certificateTransaction.outputs[0].lockingScript }) + await identityChild.sign() + + await expect(identityChild.verify(fixture.confirmedTracker)).resolves.toBe(true) + await expect(parseResults(answer(outputFor(identityChild.toBEEF())), fixture.confirmedTracker)).resolves.toEqual([]) + }) + + it('accepts a joined graph that spends distinct outputs of a shared unconfirmed ancestor', async () => { + const spendingKey = new PrivateKey(13) + const confirmedSource = new Transaction() + confirmedSource.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + const lockingScript = new P2PKH().lock(spendingKey.toAddress()) + confirmedSource.addOutput({ satoshis: 10, lockingScript }) + confirmIdentityFixtureTransaction(confirmedSource, fixture.confirmedTracker) + const ancestor = new Transaction() + ancestor.addInput({ + sourceTransaction: confirmedSource, + sourceOutputIndex: 0, + unlockingScriptTemplate: new P2PKH().unlock(spendingKey) + }) + ancestor.addOutput({ satoshis: 4, lockingScript }) + ancestor.addOutput({ satoshis: 4, lockingScript }) + await ancestor.sign() + const parents: Transaction[] = [] + for (const [sourceOutputIndex, satoshis] of [ + [0, 3], + [1, 2] + ] as const) { + const parent = new Transaction() + parent.addInput({ + sourceTransaction: ancestor, + sourceOutputIndex, + unlockingScriptTemplate: new P2PKH().unlock(spendingKey) + }) + parent.addOutput({ satoshis, lockingScript }) + await parent.sign() + parents.push(parent) + } + const joined = new Transaction() + for (const sourceTransaction of parents) { + joined.addInput({ + sourceTransaction, + sourceOutputIndex: 0, + unlockingScriptTemplate: new P2PKH().unlock(spendingKey) + }) + } + joined.addOutput({ satoshis: 4, lockingScript: fixture.certificateTransaction.outputs[0].lockingScript }) + await joined.sign() + + await expect(joined.verify(fixture.confirmedTracker)).resolves.toBe(true) + await expect(parseResults(answer(outputFor(joined.toBEEF())), fixture.confirmedTracker)).resolves.toHaveLength(1) + }) + + it.each([ + ['fractional', 0.5], + ['negative', -1], + ['out of range', 1] + ])('rejects a %s output index', async (_kind, outputIndex) => { + await expect( + parseResults(answer(outputFor(fixture.certificateBEEF, outputIndex)), fixture.confirmedTracker) + ).resolves.toEqual([]) + }) + + it('rejects missing and tampered subject envelope signatures and a signature from another subject', async () => { + const certificateJson = Utils.toArray(JSON.stringify(fixture.certificate), 'utf8') + const missingSignature = await new PushDrop(fixture.subjectWallet).lock( + [certificateJson], + IDENTITY_VERIFICATION_PROTOCOL, + IDENTITY_VERIFICATION_KEY_ID, + 'anyone', + true, + false + ) + const tamperedSignature = await new PushDrop(fixture.subjectWallet).lock( + [certificateJson], + IDENTITY_VERIFICATION_PROTOCOL, + IDENTITY_VERIFICATION_KEY_ID, + 'anyone', + true, + true + ) + const signatureChunk = tamperedSignature.chunks[3] + if (signatureChunk?.data == null) throw new Error('fixture envelope signature is missing') + signatureChunk.data = [...signatureChunk.data] + signatureChunk.data[0] ^= 1 + const wrongSubject = new ProtoWallet(new PrivateKey(14)) + const wrongKey = await new PushDrop(wrongSubject).lock( + [certificateJson], + IDENTITY_VERIFICATION_PROTOCOL, + IDENTITY_VERIFICATION_KEY_ID, + 'anyone', + true, + true + ) + + const candidates = [missingSignature, tamperedSignature, wrongKey].map(lockingScript => { + const tx = new Transaction() + tx.addOutput({ satoshis: 9, lockingScript }) + confirmIdentityFixtureTransaction(tx, fixture.confirmedTracker) + return outputFor(tx.toBEEF()) + }) + + await expect(parseResults(answer(...candidates), fixture.confirmedTracker)).resolves.toEqual([]) + }) + + it('rejects an input-free unconfirmed zero-value leaf', async () => { + const tx = new Transaction() + tx.addOutput({ satoshis: 0, lockingScript: fixture.certificateTransaction.outputs[0].lockingScript }) + + await expect(parseResults(answer(outputFor(tx.toBEEF())), fixture.confirmedTracker)).resolves.toEqual([]) + }) + + it('rejects an invalid certificate signature even when transaction and envelope verification pass', async () => { + const tamperedCertificate = { ...fixture.certificate, signature: '00'.repeat(64) } + const lockingScript = await new PushDrop(fixture.subjectWallet).lock( + [Utils.toArray(JSON.stringify(tamperedCertificate), 'utf8')], + IDENTITY_VERIFICATION_PROTOCOL, + IDENTITY_VERIFICATION_KEY_ID, + 'anyone', + true, + true + ) + const tx = new Transaction() + tx.addOutput({ satoshis: 9, lockingScript }) + confirmIdentityFixtureTransaction(tx, fixture.confirmedTracker) + + await expect(parseResults(answer(outputFor(tx.toBEEF())), fixture.confirmedTracker)).resolves.toEqual([]) + }) + + it('fails closed when SDK transaction or certificate verification returns false or throws', async () => { + const txFalse = jest.spyOn(Transaction.prototype, 'verify').mockResolvedValueOnce(false) + await expect(parseResults(answer(outputFor(fixture.certificateBEEF)), fixture.confirmedTracker)).resolves.toEqual( + [] + ) + txFalse.mockRestore() + + const txThrows = jest + .spyOn(Transaction.prototype, 'verify') + .mockRejectedValueOnce(new Error('verification unavailable')) + await expect(parseResults(answer(outputFor(fixture.certificateBEEF)), fixture.confirmedTracker)).resolves.toEqual( + [] + ) + txThrows.mockRestore() + + const certFalse = jest.spyOn(VerifiableCertificate.prototype, 'verify').mockResolvedValueOnce(false) + await expect(parseResults(answer(outputFor(fixture.certificateBEEF)), fixture.confirmedTracker)).resolves.toEqual( + [] + ) + certFalse.mockRestore() + }) + + it('fails closed when the tracker throws or returns a non-boolean verdict', async () => { + const throwingTracker = { + currentHeight: async () => IDENTITY_VERIFICATION_CONFIRMED_HEIGHT + 101, + isValidRootForHeight: async () => { + throw new Error('tracker unavailable') + } + } + const nonBooleanTracker = { + currentHeight: async () => IDENTITY_VERIFICATION_CONFIRMED_HEIGHT + 101, + isValidRootForHeight: async () => 'true' + } + + await expect(parseResults(answer(outputFor(fixture.certificateBEEF)), throwingTracker)).resolves.toEqual([]) + await expect(parseResults(answer(outputFor(fixture.certificateBEEF)), nonBooleanTracker as never)).resolves.toEqual( + [] + ) + }) + + it('fails closed without a tracker and does not query an overlay resolver without one', async () => { + const resolver = { query: jest.fn() } as unknown as LookupResolver + + await expect(parseResults(answer(outputFor(fixture.certificateBEEF)))).resolves.toEqual([]) + await expect(queryOverlay({ identityKey: fixture.certificate.subject }, resolver)).resolves.toEqual([]) + expect(resolver.query).not.toHaveBeenCalled() + }) + + it('uses the same verification path for progressive parsing and preserves a later valid candidate', async () => { + const invalidProof = Transaction.fromBEEF(fixture.certificateBEEF) + invalidProof.merklePath = new MerklePath(IDENTITY_VERIFICATION_CONFIRMED_HEIGHT, [ + [ + { offset: 0, hash: '43'.repeat(32) }, + { offset: 1, hash: invalidProof.id('hex'), txid: true } + ] + ]) + const candidates = answer(outputFor(invalidProof.toBEEF()), outputFor(fixture.certificateBEEF)) + + await expect(collect(parseResults$(candidates, fixture.confirmedTracker))).resolves.toHaveLength(1) + await expect(parseResults(candidates, fixture.confirmedTracker)).resolves.toHaveLength(1) + }) + + it('snapshots untrusted output bytes and index before the tracker can yield', async () => { + let releaseVerification: (() => void) | undefined + const gate = new Promise(resolve => { + releaseVerification = resolve + }) + const tracker = { + currentHeight: async () => IDENTITY_VERIFICATION_CONFIRMED_HEIGHT + 101, + isValidRootForHeight: async (root: string, height: number) => { + await gate + return height === IDENTITY_VERIFICATION_CONFIRMED_HEIGHT && fixture.confirmedTracker.roots.has(root) + } + } + const evidence = outputFor(fixture.certificateBEEF.slice()) + const verification = verifyOverlayOutput(evidence, tracker) + evidence.outputIndex = 1 + evidence.beef.fill(0) + releaseVerification!() + + await expect(verification).resolves.toMatchObject({ + txid: fixture.certificateTransaction.id('hex'), + outputIndex: 0 + }) + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/utility/identityUtils.ts b/packages/wallet/wallet-toolbox/src/utility/identityUtils.ts index 7518ccd22..dd6cf00e3 100644 --- a/packages/wallet/wallet-toolbox/src/utility/identityUtils.ts +++ b/packages/wallet/wallet-toolbox/src/utility/identityUtils.ts @@ -1,6 +1,5 @@ import { LookupAnswer, - Transaction, PushDrop, VerifiableCertificate, Utils, @@ -9,9 +8,11 @@ import { DiscoverCertificatesResult, IdentityCertificate, IdentityCertifier, - Base64String + Base64String, + ChainTracker } from '@bsv/sdk' import { Certifier, TrustSettings } from '../WalletSettingsManager' +import { OverlayOutputEvidence, verifyOverlayOutput } from './verifyOverlayOutput' // Our extended certificate includes certifierInfo. export interface ExtendedVerifiableCertificate extends IdentityCertificate { @@ -98,18 +99,41 @@ export const transformVerifiableCertificatesWithTrust = ( /** * Performs an identity overlay service lookup query and returns the parsed results. + * Requires an independently maintained ChainTracker; missing context returns no identities. * * Identity paths benefit from a larger grace window (more hosts contribute outputs before the * query resolves) — 300 ms is well under the "instant" perception threshold and catches the long * tail of healthy-but-slightly-slow hosts. */ -export const queryOverlay = async (query: unknown, resolver: LookupResolver): Promise => { - const results = await resolver.query({ - service: 'ls_identity', - query - }, undefined, { graceMs: 300 }) +export const queryOverlay = async ( + query: unknown, + resolver: LookupResolver, + chainTracker?: ChainTracker +): Promise => { + if (chainTracker == null) return [] + return await parseResults(await queryOverlayEvidence(query, resolver), chainTracker) +} - return await parseResults(results) +/** Fetch an owned snapshot of UNTRUSTED evidence, suitable only for revalidation. */ +export const queryOverlayEvidence = async (query: unknown, resolver: LookupResolver): Promise => { + const results = await resolver.query( + { + service: 'ls_identity', + query + }, + undefined, + { graceMs: 300 } + ) + + if (results.type !== 'output-list') return results + return { + type: 'output-list', + outputs: results.outputs.map(output => ({ + ...output, + beef: output.beef.slice(), + ...(output.context === undefined ? {} : { context: output.context.slice() }) + })) + } } /** @@ -129,7 +153,7 @@ const isUiRuntime = (): boolean => { } const yieldToUi = async (): Promise => { - await new Promise((resolve) => setTimeout(resolve, 0)) + await new Promise(resolve => setTimeout(resolve, 0)) } /** @@ -137,11 +161,12 @@ const yieldToUi = async (): Promise => { * parse / decrypt / verify failure so a malformed entry can never block the others. */ const parseOne = async ( - output: { beef: number[], outputIndex: number, context?: number[] } + output: OverlayOutputEvidence, + chainTracker: ChainTracker ): Promise => { try { - const tx = Transaction.fromBEEF(output.beef) - const decodedOutput = PushDrop.decode(tx.outputs[output.outputIndex].lockingScript) + const verifiedOutput = await verifyOverlayOutput(output, chainTracker) + const decodedOutput = PushDrop.decode(verifiedOutput.lockingScript) const certificate: VerifiableCertificate = JSON.parse(Utils.toUTF8(decodedOutput.fields[0])) const verifiableCert = new VerifiableCertificate( certificate.type, @@ -153,29 +178,48 @@ const parseOne = async ( certificate.keyring, certificate.signature ) - const decryptedFields = await verifiableCert.decryptFields(new ProtoWallet('anyone')) - await verifiableCert.verify() + // IdentityClient.publiclyRevealAttributes and tm_identity use the subject's + // BRC-42 identity key to sign the certificate/keyring fields in this output. + const anyoneWallet = new ProtoWallet('anyone') + const signature = decodedOutput.fields.pop() + if (decodedOutput.fields.length === 0 || signature == null) return null + const { valid } = await anyoneWallet.verifySignature({ + data: decodedOutput.fields.flat(), + signature, + counterparty: verifiableCert.subject, + protocolID: [1, 'identity'], + keyID: '1' + }) + if (valid !== true) return null + if ((await verifiableCert.verify()) !== true) return null + const decryptedFields = await verifiableCert.decryptFields(anyoneWallet) + if (Object.keys(decryptedFields).length === 0) return null verifiableCert.decryptedFields = decryptedFields return verifiableCert - } catch (error) { - console.error(error) + } catch { + // Untrusted parsing/decryption errors can contain identity data. Do not log it. return null } } /** * Parse the returned UTXOs, decrypting and verifying each certificate. + * An omitted ChainTracker fails closed. Each call revalidates transaction evidence; + * returned certificates carry no reusable chain-verdict or unspentness guarantee. * * On UI runtimes (browser / React Native), yields between iterations so the JS thread does not * own the frame for the full duration. On Node, runs straight through. */ -export const parseResults = async (lookupResult: LookupAnswer): Promise => { - if (lookupResult.type !== 'output-list') return [] +export const parseResults = async ( + lookupResult: LookupAnswer, + chainTracker?: ChainTracker +): Promise => { + if (lookupResult.type !== 'output-list' || chainTracker == null) return [] const parsedResults: VerifiableCertificate[] = [] const shouldYield = isUiRuntime() for (const output of lookupResult.outputs) { if (shouldYield) await yieldToUi() - const cert = await parseOne(output) + const cert = await parseOne(output, chainTracker) if (cert != null) parsedResults.push(cert) } return parsedResults @@ -185,12 +229,15 @@ export const parseResults = async (lookupResult: LookupAnswer): Promise { - if (lookupResult.type !== 'output-list') return +export async function* parseResults$( + lookupResult: LookupAnswer, + chainTracker?: ChainTracker +): AsyncIterable { + if (lookupResult.type !== 'output-list' || chainTracker == null) return const shouldYield = isUiRuntime() for (const output of lookupResult.outputs) { if (shouldYield) await yieldToUi() - const cert = await parseOne(output) + const cert = await parseOne(output, chainTracker) if (cert != null) yield cert } } diff --git a/packages/wallet/wallet-toolbox/src/utility/verifyOverlayOutput.ts b/packages/wallet/wallet-toolbox/src/utility/verifyOverlayOutput.ts new file mode 100644 index 000000000..ea2cac8be --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/utility/verifyOverlayOutput.ts @@ -0,0 +1,79 @@ +import { ChainTracker, LockingScript, Transaction } from '@bsv/sdk' + +/** Untrusted lookup evidence. A host's txid and context are never chain authority. */ +export interface OverlayOutputEvidence { + beef: number[] + outputIndex: number + txid?: string + context?: number[] +} + +/** A per-call verdict, for service validation now; not a reusable chain/cache verdict. */ +export interface VerifiedOverlayOutput { + readonly txid: string + readonly outputIndex: number + readonly lockingScript: LockingScript +} + +/** + * Independently verify a byte-bound output using the caller's canonical chain source. + * No network defaults, scripts-only mode, verdict cache, or identity semantics live here. + * Errors and false verdicts reject this evidence candidate, not its txid permanently. + */ +export async function verifyOverlayOutput( + evidence: OverlayOutputEvidence, + chainTracker: ChainTracker +): Promise { + if (chainTracker == null || typeof chainTracker.isValidRootForHeight !== 'function') { + throw new Error('Overlay verification requires a ChainTracker') + } + const { outputIndex, txid: hint } = evidence + if (!Number.isSafeInteger(outputIndex) || outputIndex < 0) { + throw new Error('Invalid overlay output index') + } + // Own the bytes before the first await: callers cannot change the candidate in flight. + const bytes = evidence.beef.slice() + if (bytes.some(byte => !Number.isInteger(byte) || byte < 0 || byte > 255)) { + throw new Error('Invalid overlay transaction bytes') + } + const tx = Transaction.fromBEEF(bytes) + const txid = tx.id('hex') + if (hint !== undefined && (typeof hint !== 'string' || hint.toLowerCase() !== txid)) { + throw new Error('Overlay txid does not match transaction bytes') + } + if (outputIndex >= tx.outputs.length) throw new Error('Overlay output does not exist') + + // SDK graph verification checks scripts and values. Also require every unconfirmed + // branch to have inputs, so a fabricated zero-value, input-free leaf cannot anchor it. + const pending = [tx] + const visited = new Set() + const spentOutpoints = new Set() + while (pending.length > 0) { + const current = pending.pop()! + const currentTxid = current.id('hex') + if (visited.has(currentTxid)) continue + visited.add(currentTxid) + if (current.merklePath != null) continue + if (current.inputs.length === 0) throw new Error('Unconfirmed transaction has no ancestry') + for (const input of current.inputs) { + if (input.sourceTransaction == null) throw new Error('Overlay transaction ancestry is missing') + const sourceTxid = input.sourceTransaction.id('hex') + if (input.sourceTXID !== undefined && input.sourceTXID !== sourceTxid) { + throw new Error('Overlay input does not match its source transaction') + } + const outpoint = `${sourceTxid}.${input.sourceOutputIndex}` + // Script verification alone can count the same value twice. Reject both + // duplicate inputs and conflicting spends among unconfirmed ancestors. + if (spentOutpoints.has(outpoint)) throw new Error('Overlay transaction graph spends an outpoint twice') + spentOutpoints.add(outpoint) + pending.push(input.sourceTransaction) + } + } + + const canonicalTracker: ChainTracker = { + currentHeight: async () => await chainTracker.currentHeight(), + isValidRootForHeight: async (root, height) => (await chainTracker.isValidRootForHeight(root, height)) === true + } + if ((await tx.verify(canonicalTracker)) !== true) throw new Error('Overlay transaction verification failed') + return { txid, outputIndex, lockingScript: tx.outputs[outputIndex].lockingScript } +} From 02c8502726176c1b1003e8119aa114f2d69d10ec Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 12:36:46 -0500 Subject: [PATCH 002/168] feat(overlay): define optional admission and recovery contracts --- .../overlays/overlay-discovery-services.md | 6 +- docs/packages/overlays/overlay-express.md | 6 +- docs/packages/overlays/overlay-topics.md | 6 +- docs/packages/overlays/overlay.md | 15 +- docs/reference/package-api-migrations.md | 24 +- docs/reference/stack-facts.md | 8 +- governance/package-release-notes.json | 16 +- governance/repository-health/baselines.json | 8 +- .../overlay-discovery-services/CHANGELOG.md | 2 + .../overlay-discovery-services/package.json | 2 +- .../overlays/overlay-express/CHANGELOG.md | 2 + .../overlays/overlay-express/package.json | 2 +- packages/overlays/overlay/CHANGELOG.md | 2 + packages/overlays/overlay/README.md | 18 + packages/overlays/overlay/mod.ts | 2 + packages/overlays/overlay/package.json | 4 +- .../src/__tests/AdmissionStorage.test.ts | 3 + .../src/__tests/StorageContract.test.ts | 168 ++++ .../admission/AdmissionStorageContract.ts | 532 +++++++++++++ .../admission/ReferenceAdmissionStorage.ts | 715 ++++++++++++++++++ .../overlay/src/storage/AdmissionStorage.ts | 254 +++++++ .../overlay/src/storage/RecoveryContract.ts | 87 +++ .../overlays/overlay/src/storage/Storage.ts | 4 + packages/overlays/topics/CHANGELOG.md | 2 + packages/overlays/topics/package.json | 2 +- specs/overlay/fixtures/persistence-v1.json | 536 +++++++++++++ specs/overlay/persistence-v1.md | 163 ++++ 27 files changed, 2544 insertions(+), 45 deletions(-) create mode 100644 packages/overlays/overlay/src/__tests/AdmissionStorage.test.ts create mode 100644 packages/overlays/overlay/src/__tests/StorageContract.test.ts create mode 100644 packages/overlays/overlay/src/__tests/admission/AdmissionStorageContract.ts create mode 100644 packages/overlays/overlay/src/__tests/admission/ReferenceAdmissionStorage.ts create mode 100644 packages/overlays/overlay/src/storage/AdmissionStorage.ts create mode 100644 packages/overlays/overlay/src/storage/RecoveryContract.ts create mode 100644 specs/overlay/fixtures/persistence-v1.json create mode 100644 specs/overlay/persistence-v1.md diff --git a/docs/packages/overlays/overlay-discovery-services.md b/docs/packages/overlays/overlay-discovery-services.md index 8400631f9..dbea79539 100644 --- a/docs/packages/overlays/overlay-discovery-services.md +++ b/docs/packages/overlays/overlay-discovery-services.md @@ -4,9 +4,9 @@ title: '@bsv/overlay-discovery-services' kind: package domain: overlays npm: '@bsv/overlay-discovery-services' -version: '2.2.1' -last_updated: '2026-08-27' -last_verified: '2026-08-27' +version: '2.2.2' +last_updated: '2026-09-08' +last_verified: '2026-09-08' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services' status: stable diff --git a/docs/packages/overlays/overlay-express.md b/docs/packages/overlays/overlay-express.md index 2c4880412..a2286439f 100644 --- a/docs/packages/overlays/overlay-express.md +++ b/docs/packages/overlays/overlay-express.md @@ -4,9 +4,9 @@ title: '@bsv/overlay-express' kind: package domain: overlays npm: '@bsv/overlay-express' -version: '2.6.1' -last_updated: '2026-08-27' -last_verified: '2026-08-27' +version: '2.6.2' +last_updated: '2026-09-08' +last_verified: '2026-09-08' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express' status: stable diff --git a/docs/packages/overlays/overlay-topics.md b/docs/packages/overlays/overlay-topics.md index 691a86814..74c0b2a85 100644 --- a/docs/packages/overlays/overlay-topics.md +++ b/docs/packages/overlays/overlay-topics.md @@ -4,9 +4,9 @@ title: '@bsv/overlay-topics' kind: package domain: overlays npm: '@bsv/overlay-topics' -version: '1.7.1' -last_updated: '2026-08-27' -last_verified: '2026-08-27' +version: '1.7.2' +last_updated: '2026-09-08' +last_verified: '2026-09-08' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics' status: stable diff --git a/docs/packages/overlays/overlay.md b/docs/packages/overlays/overlay.md index 2a4492822..8e7d462ed 100644 --- a/docs/packages/overlays/overlay.md +++ b/docs/packages/overlays/overlay.md @@ -4,9 +4,9 @@ title: '@bsv/overlay' kind: package domain: overlays npm: '@bsv/overlay' -version: '2.3.1' -last_updated: '2026-08-27' -last_verified: '2026-08-27' +version: '2.4.0' +last_updated: '2026-09-08' +last_verified: '2026-09-08' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay' status: stable @@ -76,6 +76,15 @@ const result = await engine.lookup({ - **BASM support** — BRC-136 topic anchors, TAC computation, reorg reconciliation, proof refresh, and unproven transaction maintenance +## Optional persistence contracts + +The package exports an additive `AdmissionStorage` capability, semantic identity +helpers and recovery fence/cursor predicates. These define the local durable +receipt and pending index/propagation boundary for future adapters. Current +`Engine.submit`, its early STEAK callback and Knex storage do not use the +capability. See the [persistence v1 specification](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/persistence-v1.md) +for the shared fixtures and explicit limits. No consumer migration is required. + ## Common patterns ### Implementing a TopicManager diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 995d1e809..1aea5d534 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -42,10 +42,10 @@ and clean-consumer tests remain the executable type authority. | `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | | `@bsv/message-box-client` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | -| `@bsv/overlay` | `2.2.1` | `2.3.1` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.1` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-express` | `2.5.0` | `2.6.1` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-topics` | `1.6.10` | `1.7.1` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. | +| `@bsv/overlay` | `2.2.1` | `2.4.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet, and no Mongo/runtime activation or mixed-writer database topology is included. | +| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.2` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | +| `@bsv/overlay-express` | `2.5.0` | `2.6.2` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | +| `@bsv/overlay-topics` | `1.6.10` | `1.7.2` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | | `@bsv/paymail` | `2.4.2` | `2.4.7` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. Consumers of the former bundled Money Button or Tokenized specification documents must follow the authoritative links in docs/specs/README.md. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/payment-express-middleware` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/sdk` | `2.4.0` | `2.4.2` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. | @@ -277,8 +277,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay.md](../packages/overlays/overlay.md) - Source: [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) -- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. -- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. +- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. +- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet, and no Mongo/runtime activation or mixed-writer database topology is included. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | @@ -292,8 +292,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-discovery-services.md](../packages/overlays/overlay-discovery-services.md) - Source: [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) -- Release note: Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. -- Migration: Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. +- Release note: Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. +- Migration: Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------------- | ---------------------------------------------------- | @@ -304,8 +304,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-express.md](../packages/overlays/overlay-express.md) - Source: [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) -- Release note: Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. -- Migration: Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. +- Release note: Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. +- Migration: Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------------- | ---------------------------------------------------- | @@ -316,8 +316,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-topics.md](../packages/overlays/overlay-topics.md) - Source: [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) -- Release note: Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. -- Migration: Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. +- Release note: Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. +- Migration: Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | -------------------------------------- | --------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 13c442ae0..56b67c1a9 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -58,10 +58,10 @@ authorized release action. | network | `@bsv/chirp` | `0.1.1` | browser-library | browser-bundler, browser-esm, cli, node-esm | browser, node | `>=22` | [packages/network/chirp](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/chirp) | | network | `@bsv/teranode-listener` | `1.1.5` | node-library | node-esm | node | `>=22` | [packages/network/ts-p2p](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/ts-p2p) | | overlays | `@bsv/gasp` | `1.3.6` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/overlays/gasp-core](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/gasp-core) | -| overlays | `@bsv/overlay` | `2.3.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | -| overlays | `@bsv/overlay-discovery-services` | `2.2.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | -| overlays | `@bsv/overlay-express` | `2.6.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | -| overlays | `@bsv/overlay-topics` | `1.7.1` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | +| overlays | `@bsv/overlay` | `2.4.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | +| overlays | `@bsv/overlay-discovery-services` | `2.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | +| overlays | `@bsv/overlay-express` | `2.6.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | +| overlays | `@bsv/overlay-topics` | `1.7.2` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | | sdk | `@bsv/sdk` | `2.4.2` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | | sdk | `@bsv/verifast` | `0.3.5` | wasm-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global, wasm-worker | browser, node, umd, wasm, worker | `>=22` | [packages/verifast](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/verifast) | | wallet | `@bsv/btms` | `1.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index fb190dd4d..664160c78 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -126,29 +126,29 @@ "name": "@bsv/overlay", "publishedVersion": "2.2.1", "releaseType": "minor", - "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice.", - "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package." + "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures.", + "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet, and no Mongo/runtime activation or mixed-writer database topology is included." }, { "name": "@bsv/overlay-discovery-services", "publishedVersion": "2.1.1", "releaseType": "minor", - "summary": "Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice.", - "migration": "Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package." + "summary": "Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior.", + "migration": "Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged." }, { "name": "@bsv/overlay-express", "publishedVersion": "2.5.0", "releaseType": "minor", - "summary": "Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice.", - "migration": "Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package." + "summary": "Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior.", + "migration": "Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged." }, { "name": "@bsv/overlay-topics", "publishedVersion": "1.6.10", "releaseType": "minor", - "summary": "Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name.", - "migration": "Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted." + "summary": "Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior.", + "migration": "Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged." }, { "name": "@bsv/paymail", diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index aad0934bd..3e6ca49b6 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -318,10 +318,10 @@ "@bsv/payment-express-middleware": "2.1.6", "@bsv/teranode-listener": "1.1.5", "@bsv/gasp": "1.3.6", - "@bsv/overlay": "2.3.1", - "@bsv/overlay-discovery-services": "2.2.1", - "@bsv/overlay-express": "2.6.1", - "@bsv/overlay-topics": "1.7.1", + "@bsv/overlay": "2.4.0", + "@bsv/overlay-discovery-services": "2.2.2", + "@bsv/overlay-express": "2.6.2", + "@bsv/overlay-topics": "1.7.2", "@bsv/sdk": "2.4.2", "@bsv/verifast": "0.3.5", "@bsv/btms": "1.2.2", diff --git a/packages/overlays/overlay-discovery-services/CHANGELOG.md b/packages/overlays/overlay-discovery-services/CHANGELOG.md index eb62a7421..8438ff8ba 100644 --- a/packages/overlays/overlay-discovery-services/CHANGELOG.md +++ b/packages/overlays/overlay-discovery-services/CHANGELOG.md @@ -9,6 +9,8 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +- Updates the packed workspace dependency candidate for the additive overlay persistence contract. Runtime behavior and defaults are unchanged; no consumer migration is required. + ### Added - Support `ttn` WalletAdvertiser instances with chain-isolated TerraTestNet diff --git a/packages/overlays/overlay-discovery-services/package.json b/packages/overlays/overlay-discovery-services/package.json index f03df3f92..610bb5080 100644 --- a/packages/overlays/overlay-discovery-services/package.json +++ b/packages/overlays/overlay-discovery-services/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-discovery-services", - "version": "2.2.1", + "version": "2.2.2", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/overlay-express/CHANGELOG.md b/packages/overlays/overlay-express/CHANGELOG.md index fc6b08174..ed0259671 100644 --- a/packages/overlays/overlay-express/CHANGELOG.md +++ b/packages/overlays/overlay-express/CHANGELOG.md @@ -11,6 +11,8 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +- Updates the packed workspace dependency candidate for the additive overlay persistence contract. Runtime behavior and defaults are unchanged; no consumer migration is required. + ### Added - Added the `ttn` runtime network, TTN SHIP/SLAP roots, and explicit diff --git a/packages/overlays/overlay-express/package.json b/packages/overlays/overlay-express/package.json index 9b45f02cc..a8d093072 100644 --- a/packages/overlays/overlay-express/package.json +++ b/packages/overlays/overlay-express/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-express", - "version": "2.6.1", + "version": "2.6.2", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/overlay/CHANGELOG.md b/packages/overlays/overlay/CHANGELOG.md index 8bdf170c4..e90ebaad1 100644 --- a/packages/overlays/overlay/CHANGELOG.md +++ b/packages/overlays/overlay/CHANGELOG.md @@ -20,6 +20,8 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +- Adds optional atomic admission/receipt and recovery contracts with portable identity, exact integer and fencing fixtures. Existing Engine/Knex paths are unchanged; no migration is required. + ### Added - (Include new features or significant user-visible enhancements here.) diff --git a/packages/overlays/overlay/README.md b/packages/overlays/overlay/README.md index c8c87fab5..1525ba066 100644 --- a/packages/overlays/overlay/README.md +++ b/packages/overlays/overlay/README.md @@ -79,6 +79,24 @@ The root entry point exports: imports remain available through the documented package export map, but new applications should prefer the root entry point wherever possible. +## Optional persistence capability + +`AdmissionStorage` defines an additive v1 atomic admission contract for future +adapters. `getAdmissionStorage(storage)` detects an explicit provider with both +commit and reconciliation methods. Existing Knex and injected legacy adapters +remain supported; their individual methods do not imply atomic submission. +Current `Engine.submit` does not call this capability, and its early STEAK +callback is not a durable commit receipt. + +The contract separates local commit, index visibility and propagation. It binds +operation identity to verified transaction, topic/policy and off-chain context; +uses ready payload references and outbox intents; and fences recovery by both +chain epoch and topic history generation. Helper functions and shared fixtures +pin exact integers, deterministic identity, leases and cursor eligibility. +See the [persistence specification](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/persistence-v1.md). +No consumer migration, Mongo runtime, storage-default change or database migration +is included in this release candidate. + ## Runtime and package formats The package supports both module systems: diff --git a/packages/overlays/overlay/mod.ts b/packages/overlays/overlay/mod.ts index 7c0f5a82f..7afaf8b61 100644 --- a/packages/overlays/overlay/mod.ts +++ b/packages/overlays/overlay/mod.ts @@ -6,6 +6,8 @@ export type { TopicAdmittanceContext, TopicManager } from './src/TopicManager.js // Interfaces and structures export type { Storage, AppliedTransaction } from './src/storage/Storage.js' +export * from './src/storage/AdmissionStorage.js' +export * from './src/storage/RecoveryContract.js' export type { Output } from './src/Output.js' export type { TaggedBEEF, diff --git a/packages/overlays/overlay/package.json b/packages/overlays/overlay/package.json index 7f518ae4a..1b5a200c9 100644 --- a/packages/overlays/overlay/package.json +++ b/packages/overlays/overlay/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay", - "version": "2.3.1", + "version": "2.4.0", "sideEffects": false, "engines": { "node": ">=22" @@ -81,7 +81,7 @@ "format:check": "pnpm --workspace-root exec prettier --check \"packages/overlays/overlay/{README.md,jest.config.js,mod.ts,package.json,tsconfig*.json}\"", "lint": "oxlint mod.ts src jest.config.js --deny-warnings", "lint:ci": "pnpm lint", - "pack:check": "pnpm build && node ../../../scripts/check-package-artifact.mjs . --exports BASM_ZERO_HASH,Engine,KnexStorage,KnexStorageMigrations,computeBasmRoot,computeTac,extractMerkleProofMetadata,serializeErrorForLog,serializeLogValue --entry-exports \"./storage=;./storage/knex/KnexStorage=KnexStorage\"", + "pack:check": "pnpm build && node ../../../scripts/check-package-artifact.mjs . --exports admissionSemanticDigest,canAdvanceGaspCursor,getAdmissionStorage,isRecoveryLeaseCurrent,isReplaySafeProjection,parseStorageUint64,parseStorageOutputIndex,BASM_ZERO_HASH,Engine,KnexStorage,KnexStorageMigrations,computeBasmRoot,computeTac,extractMerkleProofMetadata,serializeErrorForLog,serializeLogValue --entry-exports \"./storage=;./storage/knex/KnexStorage=KnexStorage\"", "typecheck": "tsc --project tsconfig.base.json --noEmit --incremental false", "build": "tsc -b && tsconfig-to-dual-package tsconfig.cjs.json", "dev": "tsc -b -w", diff --git a/packages/overlays/overlay/src/__tests/AdmissionStorage.test.ts b/packages/overlays/overlay/src/__tests/AdmissionStorage.test.ts new file mode 100644 index 000000000..36564abbb --- /dev/null +++ b/packages/overlays/overlay/src/__tests/AdmissionStorage.test.ts @@ -0,0 +1,3 @@ +import { referenceAdmissionStorageContract } from './admission/AdmissionStorageContract.js' + +referenceAdmissionStorageContract() diff --git a/packages/overlays/overlay/src/__tests/StorageContract.test.ts b/packages/overlays/overlay/src/__tests/StorageContract.test.ts new file mode 100644 index 000000000..a37ae19c3 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/StorageContract.test.ts @@ -0,0 +1,168 @@ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import type { Knex } from 'knex' +import { KnexStorage } from '../storage/knex/KnexStorage.js' +import { + admissionSemanticDigest, + getAdmissionStorage, + isReplaySafeProjection, + parseStorageOutputIndex, + parseStorageUint64, + type AdmissionIdentity +} from '../storage/AdmissionStorage.js' +import { + canAdvanceGaspCursor, + isRecoveryLeaseCurrent, + type GaspCursorEvidence, + type RecoveryLease +} from '../storage/RecoveryContract.js' + +// This exact language-neutral fixture is also consumed by standalone Go tests. +const fixture = JSON.parse( + readFileSync( + resolve(__dirname, '../../../../../specs/overlay/fixtures/persistence-v1.json'), + 'utf8' + ) +) as { + identities: Array<{ name: string; identity: AdmissionIdentity; digest: string }> + uint64: Array<{ value: string; valid: boolean }> + outputIndices: Array<{ value: string; valid: boolean }> + cursors: Array<{ name: string; evidence: GaspCursorEvidence; advance: boolean }> + leases: Array<{ + name: string + expected: RecoveryLease + current: RecoveryLease + nowMs: string + valid: boolean + }> +} + +describe('S01 portable persistence contract', () => { + test.each(fixture.identities)('semantic digest: $name', ({ identity, digest }) => { + expect(admissionSemanticDigest(identity)).toBe(digest) + expect(admissionSemanticDigest({ ...identity, topics: [...identity.topics].reverse() })).toBe( + digest + ) + }) + + test('semantic identity binds all immutable admission inputs', () => { + const original = fixture.identities[0].identity + const originalDigest = admissionSemanticDigest(original) + const variants: AdmissionIdentity[] = [ + { ...original, txid: '2'.repeat(64) }, + { ...original, mode: 'historical' }, + { ...original, contextDigest: '3'.repeat(64) }, + { ...original, scope: { ...original.scope, network: 'main' } }, + { ...original, scope: { ...original.scope, genesisHash: '4'.repeat(64) } }, + { ...original, scope: { ...original.scope, nodeId: 'node-b' } }, + { ...original, topics: [{ topic: 'tm_a', policyId: 'different-policy' }] } + ] + for (const identity of variants) + expect(admissionSemanticDigest(identity)).not.toBe(originalDigest) + }) + + test('rejects duplicate topics, empty identity and ambiguous Unicode before hashing', () => { + const original = fixture.identities[0].identity + for (const topics of [ + [], + [original.topics[0], original.topics[0]], + [{ topic: '\ud800', policyId: 'p' }], + [{ topic: 't', policyId: '' }] + ]) { + expect(() => admissionSemanticDigest({ ...original, topics })).toThrow() + } + expect(() => admissionSemanticDigest({ ...original, txid: 'A'.repeat(64) })).toThrow( + 'Invalid admission hash' + ) + expect(() => + admissionSemanticDigest({ ...original, scope: { ...original.scope, genesisHash: 'invalid' } }) + ).toThrow() + expect(() => admissionSemanticDigest({ ...original, contextDigest: 'invalid' })).toThrow() + expect(() => + admissionSemanticDigest({ ...original, mode: 'invalid' as AdmissionIdentity['mode'] }) + ).toThrow('Invalid admission mode') + }) + + test.each(fixture.uint64)('exact uint64: "$value"', ({ value, valid }) => { + if (valid) expect(parseStorageUint64(value).toString()).toBe(value) + else expect(() => parseStorageUint64(value)).toThrow('Invalid storage uint64') + }) + + test('does not coerce a JavaScript number at a runtime boundary', () => { + expect(() => parseStorageUint64(1 as unknown as string)).toThrow() + }) + + test.each(fixture.outputIndices)('exact output index: "$value"', ({ value, valid }) => { + if (valid) expect(String(parseStorageOutputIndex(value))).toBe(value) + else expect(() => parseStorageOutputIndex(value)).toThrow() + }) + + test.each(fixture.cursors)('cursor publication: $name', ({ evidence, advance }) => { + expect(canAdvanceGaspCursor(evidence)).toBe(advance) + }) + + test.each(fixture.leases)('recovery fence: $name', ({ expected, current, nowMs, valid }) => { + expect(isRecoveryLeaseCurrent(expected, current, nowMs)).toBe(valid) + }) + + test('fences topic, job and genesis as well as node/peer and generations', () => { + const original = fixture.leases[0].current + for (const current of [ + { ...original, topic: 'tm_other' }, + { ...original, jobId: 'other-job' }, + { ...original, scope: { ...original.scope, genesisHash: 'a'.repeat(64) } }, + { ...original, scope: { ...original.scope, network: 'other-network' } } + ]) + expect(isRecoveryLeaseCurrent(original, current, '999')).toBe(false) + expect(() => + isRecoveryLeaseCurrent(original, { ...original, chainEpoch: '01' }, '999') + ).toThrow() + }) + + test('legacy Knex CRUD cannot advertise atomic admission', () => { + const knex = jest.fn() as unknown as Knex + expect(getAdmissionStorage(new KnexStorage(knex))).toBeUndefined() + expect(knex).not.toHaveBeenCalled() + }) + + test('requires explicit v1 plus both commit and reconciliation functions', () => { + const admission = { + protocol: 'overlay-admission-v1', + commitAdmission: jest.fn(), + reconcileAdmission: jest.fn() + } + for (const storage of [ + null, + undefined, + 1, + {}, + { admission: null }, + { admission: {} }, + { admission: { ...admission, protocol: 'overlay-admission-v2' } }, + { admission: { ...admission, commitAdmission: undefined } }, + { admission: { ...admission, reconcileAdmission: undefined } } + ]) { + expect(getAdmissionStorage(storage)).toBeUndefined() + } + expect(getAdmissionStorage({ admission })).toBe(admission) + expect(admission.commitAdmission).not.toHaveBeenCalled() + }) + + test('legacy lookup callbacks never imply replay or reconciliation safety', () => { + const projection = { + protocol: 'overlay-projection-v1', + applyEvent: jest.fn(), + reconcile: jest.fn() + } + expect(isReplaySafeProjection(projection)).toBe(true) + for (const value of [ + null, + {}, + { outputAdmittedByTopic: jest.fn() }, + { ...projection, protocol: 'other' }, + { ...projection, applyEvent: null }, + { ...projection, reconcile: undefined } + ]) + expect(isReplaySafeProjection(value)).toBe(false) + }) +}) diff --git a/packages/overlays/overlay/src/__tests/admission/AdmissionStorageContract.ts b/packages/overlays/overlay/src/__tests/admission/AdmissionStorageContract.ts new file mode 100644 index 000000000..320f28e84 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/admission/AdmissionStorageContract.ts @@ -0,0 +1,532 @@ +import { + admissionSemanticDigest, + getAdmissionStorage, + type AdmissionCommit, + type AdmissionCommitResult, + type AdmissionPayloadRef, + type AdmissionReceipt, + type AdmissionReconcileResult, + type ReplaySafeProjection +} from '../../storage/AdmissionStorage.js' +import type { RecoveryLease } from '../../storage/RecoveryContract.js' +import { + ReferenceAdmissionHarness, + referenceScope, + type AdmissionStorageContractHarness +} from './ReferenceAdmissionStorage.js' + +const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T +const hash = (character: string): string => character.repeat(64) + +const history = { chainEpoch: '7', topicHistoryGeneration: '3' } +const source = { txid: hash('9'), outputIndex: '0' } +const raw: AdmissionPayloadRef = { digest: hash('a'), byteLength: '100', kind: 'raw-transaction' } +const script: AdmissionPayloadRef = { digest: hash('b'), byteLength: '25', kind: 'locking-script' } +const proof: AdmissionPayloadRef = { digest: hash('c'), byteLength: '40', kind: 'merkle-path' } +const outboxData: AdmissionPayloadRef = { digest: hash('d'), byteLength: '10', kind: 'outbox-data' } +const alternateProof: AdmissionPayloadRef = { + digest: hash('e'), + byteLength: '41', + kind: 'merkle-path' +} +const steakFor = (topic: string): string => + JSON.stringify({ [topic]: { outputsToAdmit: [0], coinsToRetain: [], coinsRemoved: [] } }) + +export const admissionPlan = (operationId = 'operation-1', txid = hash('8')): AdmissionCommit => { + const identity = { + scope: referenceScope, + txid, + mode: 'live' as const, + contextDigest: hash('2'), + topics: [{ topic: 'tm_contract', policyId: 'policy-1' }] + } + return { + key: { scope: referenceScope, operationId, semanticDigest: admissionSemanticDigest(identity) }, + identity, + payloads: [raw, script, proof, outboxData], + decisions: [ + { + topic: 'tm_contract', + expectedHistory: history, + reads: [{ key: 'selection:tm_contract', expectedVersion: 'r1' }], + spends: [{ outpoint: source, expectedVersion: 'spend-r1', spender: txid }], + evictions: [], + outputs: [ + { + txid, + outputIndex: '0', + satoshis: '1', + score: '10', + script: { payload: script, offset: '0', byteLength: '25' } + } + ], + edges: [{ source, consumer: { txid, outputIndex: '0' } }], + applied: { + txid, + firstSeenHeight: '100', + proof, + block: { height: '101', hash: hash('3'), index: '0', merkleRoot: hash('4') } + } + } + ], + outbox: [ + { + eventId: `${operationId}:lookup`, + kind: 'lookup', + target: 'ls_contract', + payloads: [outboxData] + }, + { + eventId: `${operationId}:propagate`, + kind: 'propagation', + target: 'peer-a', + payloads: [raw] + } + ], + steak: steakFor('tm_contract') + } +} + +const planForScopeAndTopic = ( + scope: typeof referenceScope, + topic: string, + operationId: string +): AdmissionCommit => { + const plan = admissionPlan(operationId) + plan.identity.scope = scope + plan.identity.topics = [{ topic, policyId: 'policy-1' }] + plan.key.scope = scope + plan.key.semanticDigest = admissionSemanticDigest(plan.identity) + plan.decisions[0].topic = topic + plan.decisions[0].reads = [{ key: `selection:${topic}`, expectedVersion: 'r1' }] + plan.steak = steakFor(topic) + return plan +} + +const seedPlan = async ( + harness: AdmissionStorageContractHarness, + plan: AdmissionCommit, + omit: string[] = [] +): Promise => { + for (const decision of plan.decisions) { + await harness.seed.history(plan.identity.scope, decision.topic, decision.expectedHistory) + for (const read of decision.reads) { + if (read.expectedVersion !== null) { + await harness.seed.read(plan.identity.scope, decision.topic, read.key, read.expectedVersion) + } + } + for (const spend of decision.spends) { + await harness.seed.spendable( + plan.identity.scope, + decision.topic, + spend.outpoint, + spend.expectedVersion + ) + } + } + for (const payload of plan.payloads) { + if (!omit.includes(payload.digest)) await harness.seed.readyPayload(payload) + } +} + +const committedReceipt = ( + result: AdmissionCommitResult | AdmissionReconcileResult +): AdmissionReceipt => { + if (result.state !== 'committed') + throw new Error(`Expected committed result, got ${result.state}`) + return result.receipt +} + +const rejected = ( + result: AdmissionCommitResult, + code: Extract['code'] +): void => { + expect(result).toEqual({ state: 'rejected', code }) +} + +/** + * Reusable behavioral contract for an AdmissionStorage adapter. The reference + * implementation is an in-memory model only; passing this suite does not + * establish database durability, crash recovery, or external delivery. + */ +export const admissionStorageContract = ( + createHarness: () => AdmissionStorageContractHarness +): void => { + describe('AdmissionStorage contract', () => { + let harness: AdmissionStorageContractHarness + + beforeEach(async () => { + harness = createHarness() + await harness.reset() + }) + + test('advertises the optional admission provider protocol', () => { + expect(getAdmissionStorage(harness.storage)?.protocol).toBe('overlay-admission-v1') + }) + + test('commits one atomic admission and leaves external work pending', async () => { + const plan = admissionPlan() + await seedPlan(harness, plan) + + const receipt = committedReceipt(await harness.storage.admission!.commitAdmission(plan)) + + expect(receipt).toEqual({ + operationId: plan.key.operationId, + semanticDigest: plan.key.semanticDigest, + durability: 'atomic-local', + steak: plan.steak, + indexes: [{ target: 'ls_contract', state: 'pending' }], + propagation: 'pending' + }) + const snapshot = await harness.snapshot() + expect(snapshot.payloads.every(payload => payload.pinned)).toBe(true) + expect(snapshot.outputs.some(output => output.spentBy === plan.identity.txid)).toBe(true) + expect(snapshot.edges).toHaveLength(1) + expect(snapshot.applied).toEqual([ + expect.objectContaining({ record: expect.objectContaining({ txid: plan.identity.txid }) }) + ]) + expect(snapshot.outbox).toEqual([ + { scope: referenceScope, eventId: `${plan.key.operationId}:lookup` }, + { scope: referenceScope, eventId: `${plan.key.operationId}:propagate` } + ]) + }) + + test('returns the saved receipt for concurrent and restart retries before stale predicates', async () => { + const plan = admissionPlan() + await seedPlan(harness, plan) + + const [first, second] = await Promise.all([ + harness.storage.admission!.commitAdmission(plan), + harness.storage.admission!.commitAdmission(clone(plan)) + ]) + const saved = committedReceipt(first) + expect(committedReceipt(second)).toEqual(saved) + + saved.steak = 'caller mutation' + saved.indexes[0].state = 'visible' + await harness.seed.read(referenceScope, 'tm_contract', 'selection:tm_contract', 'r2') + await harness.seed.readyPayload(alternateProof) + const alternate = clone(plan) + alternate.payloads = [raw, script, alternateProof, outboxData] + alternate.decisions[0].applied.proof = alternateProof + alternate.steak = 'not regenerated on retry' + + expect(committedReceipt(await harness.restart().commitAdmission(alternate))).toEqual({ + operationId: plan.key.operationId, + semanticDigest: plan.key.semanticDigest, + durability: 'atomic-local', + steak: plan.steak, + indexes: [{ target: 'ls_contract', state: 'pending' }], + propagation: 'pending' + }) + }) + + test('rejects reuse of an operation ID with another semantic digest', async () => { + const plan = admissionPlan() + await seedPlan(harness, plan) + await harness.storage.admission!.commitAdmission(plan) + + const conflicting = admissionPlan(plan.key.operationId, hash('7')) + rejected(await harness.storage.admission!.commitAdmission(conflicting), 'digest-mismatch') + }) + + test('rejects a supplied digest that does not bind the identity without publishing effects', async () => { + const plan = admissionPlan() + await seedPlan(harness, plan) + const before = await harness.snapshot() + plan.key.semanticDigest = hash('0') + + rejected(await harness.storage.admission!.commitAdmission(plan), 'digest-mismatch') + expect(await harness.snapshot()).toEqual(before) + }) + + test('rejects a missing ready payload without partially applying the plan', async () => { + const plan = admissionPlan() + await seedPlan(harness, plan, [proof.digest]) + const before = await harness.snapshot() + + rejected(await harness.storage.admission!.commitAdmission(plan), 'payload-not-ready') + expect(await harness.snapshot()).toEqual(before) + }) + + test('rejects unsupported semantic effects before publishing a partial admission', async () => { + const wrongOutput = admissionPlan() + await seedPlan(harness, wrongOutput) + const before = await harness.snapshot() + wrongOutput.decisions[0].outputs[0].txid = hash('7') + rejected(await harness.storage.admission!.commitAdmission(wrongOutput), 'invalid-plan') + expect(await harness.snapshot()).toEqual(before) + + const historicalPropagation = admissionPlan('historical-operation') + historicalPropagation.identity.mode = 'historical' + historicalPropagation.key.semanticDigest = admissionSemanticDigest( + historicalPropagation.identity + ) + await seedPlan(harness, historicalPropagation) + rejected( + await harness.storage.admission!.commitAdmission(historicalPropagation), + 'invalid-plan' + ) + expect(await harness.snapshot()).toEqual(before) + }) + + test('rejects non-wire output indexes and duplicate decision topics without effects', async () => { + const oversized = admissionPlan('oversized-index') + await seedPlan(harness, oversized) + const before = await harness.snapshot() + oversized.decisions[0].outputs[0].outputIndex = '9007199254740993' + oversized.decisions[0].edges[0].consumer.outputIndex = '9007199254740993' + // The old model rounded the index and accepted precisely this incorrect ACK. + oversized.steak = JSON.stringify({ + tm_contract: { outputsToAdmit: [9007199254740992], coinsToRetain: [], coinsRemoved: [] } + }) + rejected(await harness.storage.admission!.commitAdmission(oversized), 'invalid-plan') + expect(await harness.snapshot()).toEqual(before) + + const uint32Overflow = admissionPlan('uint32-overflow') + await seedPlan(harness, uint32Overflow) + uint32Overflow.decisions[0].outputs[0].outputIndex = '4294967296' + uint32Overflow.decisions[0].edges[0].consumer.outputIndex = '4294967296' + uint32Overflow.steak = JSON.stringify({ + tm_contract: { outputsToAdmit: [4294967296], coinsToRetain: [], coinsRemoved: [] } + }) + rejected(await harness.storage.admission!.commitAdmission(uint32Overflow), 'invalid-plan') + expect(await harness.snapshot()).toEqual(before) + + const duplicateTopic = admissionPlan('duplicate-decision') + duplicateTopic.identity.topics.push({ topic: 'tm_other', policyId: 'policy-2' }) + duplicateTopic.key.semanticDigest = admissionSemanticDigest(duplicateTopic.identity) + duplicateTopic.decisions[0].outputs = [] + duplicateTopic.decisions[0].spends = [] + duplicateTopic.decisions[0].edges = [] + duplicateTopic.decisions.push(clone(duplicateTopic.decisions[0])) + // Matching key count and no duplicate output insertion must not hide the omitted topic. + duplicateTopic.steak = JSON.stringify({ + tm_contract: { outputsToAdmit: [], coinsToRetain: [], coinsRemoved: [] }, + tm_other: { outputsToAdmit: [77], coinsToRetain: [], coinsRemoved: [] } + }) + await seedPlan(harness, duplicateTopic) + rejected(await harness.storage.admission!.commitAdmission(duplicateTopic), 'invalid-plan') + expect(await harness.snapshot()).toEqual(before) + }) + + test('rejects stale read and conditional-spend predicates without partially applying the plan', async () => { + const staleRead = admissionPlan() + await seedPlan(harness, staleRead) + await harness.seed.read(referenceScope, 'tm_contract', 'selection:tm_contract', 'r2') + const readBefore = await harness.snapshot() + rejected(await harness.storage.admission!.commitAdmission(staleRead), 'read-conflict') + expect(await harness.snapshot()).toEqual(readBefore) + + await harness.reset() + const staleSpend = admissionPlan() + await seedPlan(harness, staleSpend) + await harness.seed.spendable(referenceScope, 'tm_contract', source, 'spend-r2') + const spendBefore = await harness.snapshot() + rejected(await harness.storage.admission!.commitAdmission(staleSpend), 'spend-conflict') + expect(await harness.snapshot()).toEqual(spendBefore) + }) + + test('allows only one concurrent conditional spend and publishes no loser effects', async () => { + const first = admissionPlan('race-first', hash('6')) + const second = admissionPlan('race-second', hash('7')) + await seedPlan(harness, first) + await seedPlan(harness, second) + + const results = await Promise.all([ + harness.storage.admission!.commitAdmission(first), + harness.storage.admission!.commitAdmission(second) + ]) + expect(results.filter(result => result.state === 'committed')).toHaveLength(1) + expect( + results.filter(result => result.state === 'rejected' && result.code === 'spend-conflict') + ).toHaveLength(1) + + const winner = results.find(result => result.state === 'committed') + if (winner?.state !== 'committed') throw new Error('Expected a single committed spender') + const winnerTxid = + winner.receipt.operationId === first.key.operationId + ? first.identity.txid + : second.identity.txid + const snapshot = await harness.snapshot() + expect(snapshot.outputs.filter(output => output.output?.txid === winnerTxid)).toHaveLength(1) + expect(snapshot.applied).toEqual([ + expect.objectContaining({ record: expect.objectContaining({ txid: winnerTxid }) }) + ]) + expect( + snapshot.outbox.every(event => + event.eventId.startsWith( + winnerTxid === first.identity.txid ? 'race-first:' : 'race-second:' + ) + ) + ).toBe(true) + }) + + test('isolates same outpoints and outbox event IDs by scope and topic', async () => { + const otherScope = { ...referenceScope, nodeId: 'reference-node-b' } + const first = planForScopeAndTopic(referenceScope, 'tm_contract', 'shared-operation') + const second = planForScopeAndTopic(otherScope, 'tm_contract', 'shared-operation') + const third = planForScopeAndTopic(referenceScope, 'tm_other', 'topic-operation') + second.decisions[0].reads[0].expectedVersion = 'r2' + await seedPlan(harness, first) + await seedPlan(harness, second) + await seedPlan(harness, third) + + committedReceipt(await harness.storage.admission!.commitAdmission(first)) + committedReceipt(await harness.storage.admission!.commitAdmission(second)) + committedReceipt(await harness.storage.admission!.commitAdmission(third)) + + const snapshot = await harness.snapshot() + expect( + snapshot.outputs.filter(output => output.output?.txid === first.identity.txid) + ).toHaveLength(3) + expect(snapshot.outbox.filter(event => event.eventId === 'shared-operation:lookup')).toEqual([ + { scope: referenceScope, eventId: 'shared-operation:lookup' }, + { scope: otherScope, eventId: 'shared-operation:lookup' } + ]) + }) + + test('does not overwrite a scope-local outbox event from another operation', async () => { + const first = admissionPlan('outbox-first') + await seedPlan(harness, first) + committedReceipt(await harness.storage.admission!.commitAdmission(first)) + const before = await harness.snapshot() + + const conflicting = admissionPlan('outbox-second', hash('6')) + conflicting.decisions[0].spends = [] + conflicting.outbox = clone(first.outbox) + rejected(await harness.storage.admission!.commitAdmission(conflicting), 'invalid-plan') + expect(await harness.snapshot()).toEqual(before) + }) + + test('moves a history revision and recovery checkpoint atomically, rejecting a stale worker', async () => { + const plan = admissionPlan() + await seedPlan(harness, plan) + const lease: RecoveryLease = { + scope: referenceScope, + topic: 'tm_contract', + peerId: 'peer-a', + jobId: 'repair-a', + leaseToken: '9', + expiresAtMs: '100', + ...history + } + await harness.seed.lease(lease) + await harness.seed.now('50') + plan.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '4', + affectedFromHeight: '99', + handoff: { expected: lease, checkpoint: 'repair-checkpoint' } + } + + committedReceipt(await harness.storage.admission!.commitAdmission(plan)) + const snapshot = await harness.snapshot() + expect(snapshot.fences.find(item => item.topic === 'tm_contract')?.fence).toEqual({ + chainEpoch: '7', + topicHistoryGeneration: '4' + }) + expect(snapshot.handoffs).toEqual([{ topic: 'tm_contract', checkpoint: 'repair-checkpoint' }]) + expect(snapshot.historyUpdates).toEqual([ + { topic: 'tm_contract', affectedFromHeight: '99', checkpoint: 'repair-checkpoint' } + ]) + expect(snapshot.leases).toEqual([ + expect.objectContaining({ ...lease, topicHistoryGeneration: '4' }) + ]) + + const stale = admissionPlan('operation-2', hash('6')) + await seedPlan(harness, stale) + await harness.seed.history(referenceScope, 'tm_contract', { + chainEpoch: '7', + topicHistoryGeneration: '4' + }) + stale.decisions[0].expectedHistory = { chainEpoch: '7', topicHistoryGeneration: '4' } + stale.decisions[0].spends = [] + stale.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '5', + affectedFromHeight: '99', + handoff: { expected: lease, checkpoint: 'stale-checkpoint' } + } + const staleBefore = await harness.snapshot() + rejected(await harness.storage.admission!.commitAdmission(stale), 'read-conflict') + expect(await harness.snapshot()).toEqual(staleBefore) + }) + + test('recovers a lost entire post-commit response before running a new body', async () => { + const plan = admissionPlan() + await seedPlan(harness, plan) + await harness.faults.loseReplyAfterCommitOnce() + + await harness.storage.admission!.commitAdmission(plan) + const afterCommit = await harness.snapshot() + + const recovered = await harness.restart().commitAdmission(clone(plan)) + expect(recovered.state).toBe('pending') + if (recovered.state !== 'pending') + throw new Error('Expected a recovered opaque commit attempt') + expect(await harness.restart().reconcileAdmission(plan.key, 'wrong-attempt')).toEqual( + recovered + ) + expect(await harness.snapshot()).toEqual(afterCommit) + expect(committedReceipt(await harness.restart().reconcileAdmission(plan.key))).toEqual({ + operationId: plan.key.operationId, + semanticDigest: plan.key.semanticDigest, + durability: 'atomic-local', + steak: plan.steak, + indexes: [{ target: 'ls_contract', state: 'pending' }], + propagation: 'pending' + }) + expect(await harness.snapshot()).toEqual(afterCommit) + }) + + test('publishes no effects when an attempt is definitively aborted before commit', async () => { + const plan = admissionPlan() + await seedPlan(harness, plan) + const before = await harness.snapshot() + await harness.faults.abortBeforeCommitOnce() + + const pending = await harness.storage.admission!.commitAdmission(plan) + expect(pending.state).toBe('pending') + if (pending.state !== 'pending') throw new Error('Expected an opaque abort attempt') + expect(await harness.snapshot()).toEqual(before) + expect(await harness.storage.admission!.commitAdmission(plan)).toEqual(pending) + expect( + await harness.storage.admission!.reconcileAdmission(plan.key, pending.attemptId) + ).toEqual({ state: 'aborted' }) + expect(await harness.snapshot()).toEqual(before) + expect(committedReceipt(await harness.storage.admission!.commitAdmission(plan)).steak).toBe( + plan.steak + ) + }) + }) +} + +export const referenceAdmissionStorageContract = (): void => { + admissionStorageContract(() => new ReferenceAdmissionHarness()) +} + +describe('reference adapter boundaries', () => { + test('rejects a non-replay-safe projection before running the admission body', async () => { + const harness = new ReferenceAdmissionHarness({ projector: { protocol: 'not-replay-safe' } }) + const plan = admissionPlan() + await seedPlan(harness, plan) + const before = harness.snapshot() + + rejected(await harness.storage.admission!.commitAdmission(plan), 'unsupported-projection') + expect(harness.snapshot()).toEqual(before) + }) + + test('does not invoke even a replay-safe projector during model commit', async () => { + const projector: ReplaySafeProjection = { + protocol: 'overlay-projection-v1', + applyEvent: jest.fn(async () => ({ checkpoint: 'unexpected' })), + reconcile: jest.fn(async () => ({ checkpoint: 'unexpected' })) + } + const harness = new ReferenceAdmissionHarness({ projector }) + const plan = admissionPlan() + await seedPlan(harness, plan) + + committedReceipt(await harness.storage.admission!.commitAdmission(plan)) + expect(projector.applyEvent).not.toHaveBeenCalled() + expect(projector.reconcile).not.toHaveBeenCalled() + }) +}) diff --git a/packages/overlays/overlay/src/__tests/admission/ReferenceAdmissionStorage.ts b/packages/overlays/overlay/src/__tests/admission/ReferenceAdmissionStorage.ts new file mode 100644 index 000000000..37142b9ce --- /dev/null +++ b/packages/overlays/overlay/src/__tests/admission/ReferenceAdmissionStorage.ts @@ -0,0 +1,715 @@ +import { + admissionSemanticDigest, + isReplaySafeProjection, + parseStorageOutputIndex, + parseStorageUint64, + type AdmissionCommit, + type AdmissionCommitResult, + type AdmissionOperationKey, + type AdmissionOutboxIntent, + type AdmissionOutput, + type AdmissionOutpoint, + type AdmissionPayloadRef, + type AdmissionReceipt, + type AdmissionReconcileResult, + type AdmissionStorage, + type AdmissionTopicDecision, + type HistoryFence, + type ReplaySafeProjection, + type StorageScope +} from '../../storage/AdmissionStorage.js' +import { + isRecoveryLeaseCurrent, + type HistoryRevisionHandoff, + type RecoveryLease +} from '../../storage/RecoveryContract.js' + +type StoredOutput = { + version: string + spentBy?: string + topic?: string + output?: AdmissionOutput +} + +type SavedOperation = { + semanticDigest: string + receipt: AdmissionReceipt +} + +type Attempt = { + key: string + semanticDigest: string + state: 'unknown' | 'aborted' +} + +type ReferenceState = { + payloads: Map + pins: Set + reads: Map + fences: Map + leases: Map + outputs: Map + edges: Set + applied: Map + outbox: Map + handoffs: Map + historyUpdates: Map + operations: Map + attempts: Map + nextAttempt: number + nowMs: string +} + +export interface AdmissionStorageTestSnapshot { + payloads: Array<{ ref: AdmissionPayloadRef; pinned: boolean }> + reads: Array<{ key: string; version: string }> + fences: Array<{ topic: string; fence: HistoryFence }> + leases: RecoveryLease[] + outputs: Array<{ + key: string + version: string + spentBy?: string + topic?: string + output?: AdmissionOutput + }> + edges: string[] + applied: Array<{ key: string; record: AdmissionTopicDecision['applied'] }> + outbox: Array<{ scope: StorageScope; eventId: string }> + handoffs: Array<{ topic: string; checkpoint: string }> + historyUpdates: Array<{ topic: string; affectedFromHeight: string; checkpoint?: string }> + operations: Array<{ key: string; semanticDigest: string }> +} + +export interface AdmissionStorageFaults { + /** The transaction commits, but the caller must reconcile its opaque attempt. */ + loseReplyAfterCommitOnce: () => void | Promise + /** The attempt becomes definitively aborted before any state is published. */ + abortBeforeCommitOnce: () => void | Promise +} + +export interface AdmissionStorageSeeds { + readyPayload: (ref: AdmissionPayloadRef) => void | Promise + read: (scope: StorageScope, topic: string, key: string, version: string) => void | Promise + spendable: ( + scope: StorageScope, + topic: string, + outpoint: AdmissionOutpoint, + version: string + ) => void | Promise + history: (scope: StorageScope, topic: string, fence: HistoryFence) => void | Promise + lease: (lease: RecoveryLease) => void | Promise + now: (milliseconds: string) => void | Promise +} + +/** + * Test harness shape that a durable adapter can implement to run the contract + * suite. Snapshot data deliberately describes observable fixtures only; it is + * not a persistence schema. + */ +export interface AdmissionStorageContractHarness { + readonly storage: { admission?: AdmissionStorage } + reset: () => void | Promise + restart: () => AdmissionStorage + snapshot: () => AdmissionStorageTestSnapshot | Promise + readonly seed: AdmissionStorageSeeds + readonly faults: AdmissionStorageFaults +} + +export interface ReferenceAdmissionStorageOptions { + /** A projection is only accepted when it declares replay/reconcile support. */ + projector?: unknown +} + +const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T + +const scopeKey = (scope: StorageScope): string => + JSON.stringify([scope.network, scope.genesisHash, scope.nodeId]) +const operationKey = (key: AdmissionOperationKey): string => + JSON.stringify([scopeKey(key.scope), key.operationId]) +const topicKey = (scope: StorageScope, topic: string): string => + JSON.stringify([scopeKey(scope), topic]) +const outputKey = (scope: StorageScope, topic: string, outpoint: AdmissionOutpoint): string => + JSON.stringify([scopeKey(scope), topic, outpoint.txid, outpoint.outputIndex]) +const readKey = (scope: StorageScope, topic: string, key: string): string => + JSON.stringify([scopeKey(scope), topic, key]) +const outboxKey = (scope: StorageScope, eventId: string): string => + JSON.stringify([scope.network, scope.genesisHash, scope.nodeId, eventId]) +const leaseKey = (lease: RecoveryLease): string => + JSON.stringify([ + lease.scope.network, + lease.scope.genesisHash, + lease.scope.nodeId, + lease.topic, + lease.peerId, + lease.jobId + ]) +const edgeKey = ( + scope: StorageScope, + topic: string, + source: AdmissionOutpoint, + consumer: AdmissionOutpoint +): string => + JSON.stringify([scope.network, scope.genesisHash, scope.nodeId, topic, source, consumer]) +const appliedKey = (scope: StorageScope, topic: string, txid: string): string => + JSON.stringify([scope.network, scope.genesisHash, scope.nodeId, topic, txid]) +const sameScope = (left: StorageScope, right: StorageScope): boolean => + scopeKey(left) === scopeKey(right) +const sameFence = (left: HistoryFence | undefined, right: HistoryFence): boolean => + left?.chainEpoch === right.chainEpoch && + left.topicHistoryGeneration === right.topicHistoryGeneration +const samePayload = (left: AdmissionPayloadRef | undefined, right: AdmissionPayloadRef): boolean => + left !== undefined && + left.digest === right.digest && + left.byteLength === right.byteLength && + left.kind === right.kind +const isHash = (value: string): boolean => /^[0-9a-f]{64}$/.test(value) +const isUint64 = (value: string): boolean => { + try { + parseStorageUint64(value) + return true + } catch { + return false + } +} +const isWireOutpoint = (outpoint: AdmissionOutpoint): boolean => { + if (!isHash(outpoint.txid)) return false + try { + parseStorageOutputIndex(outpoint.outputIndex) + return true + } catch { + return false + } +} + +const newState = (): ReferenceState => ({ + payloads: new Map(), + pins: new Set(), + reads: new Map(), + fences: new Map(), + leases: new Map(), + outputs: new Map(), + edges: new Set(), + applied: new Map(), + outbox: new Map(), + handoffs: new Map(), + historyUpdates: new Map(), + operations: new Map(), + attempts: new Map(), + nextAttempt: 1, + nowMs: '0' +}) + +const copyState = (state: ReferenceState): ReferenceState => ({ + payloads: new Map([...state.payloads].map(([key, value]) => [key, clone(value)])), + pins: new Set(state.pins), + reads: new Map(state.reads), + fences: new Map([...state.fences].map(([key, value]) => [key, clone(value)])), + leases: new Map([...state.leases].map(([key, value]) => [key, clone(value)])), + outputs: new Map([...state.outputs].map(([key, value]) => [key, clone(value)])), + edges: new Set(state.edges), + applied: new Map([...state.applied].map(([key, value]) => [key, clone(value)])), + outbox: new Map([...state.outbox].map(([key, value]) => [key, clone(value)])), + handoffs: new Map(state.handoffs), + historyUpdates: new Map([...state.historyUpdates].map(([key, value]) => [key, clone(value)])), + operations: new Map([...state.operations].map(([key, value]) => [key, clone(value)])), + attempts: new Map([...state.attempts].map(([key, value]) => [key, clone(value)])), + nextAttempt: state.nextAttempt, + nowMs: state.nowMs +}) + +/** + * A deliberately small in-memory model for admission-contract tests. It is + * shared by instances to model restart replay, but proves neither database + * durability nor crash recovery of a production adapter. + */ +export class ReferenceAdmissionStorage implements AdmissionStorage { + readonly protocol = 'overlay-admission-v1' as const + + constructor( + private readonly shared: { state: ReferenceState }, + private readonly faults: { loseReplyAfterCommit: boolean; abortBeforeCommit: boolean }, + private readonly options: ReferenceAdmissionStorageOptions = {} + ) {} + + async commitAdmission(plan: AdmissionCommit): Promise { + const semanticDigest = this.planSemanticDigest(plan) + if ( + semanticDigest === undefined || + plan.key.semanticDigest !== semanticDigest || + !sameScope(plan.key.scope, plan.identity.scope) + ) { + return { state: 'rejected', code: 'digest-mismatch' } + } + + const key = operationKey(plan.key) + const active = this.shared.state.attempts.get(key) + const saved = this.shared.state.operations.get(key) + if (saved !== undefined && saved.semanticDigest !== semanticDigest) { + return { state: 'rejected', code: 'digest-mismatch' } + } + if (active !== undefined) { + return active.semanticDigest === semanticDigest + ? { state: 'pending', attemptId: active.key } + : { state: 'rejected', code: 'digest-mismatch' } + } + if (saved !== undefined) return { state: 'committed', receipt: clone(saved.receipt) } + if (this.options.projector !== undefined && !isReplaySafeProjection(this.options.projector)) { + return { state: 'rejected', code: 'unsupported-projection' } + } + + if (this.faults.abortBeforeCommit) { + this.faults.abortBeforeCommit = false + const attemptId = this.newAttempt(key, semanticDigest, 'aborted') + return { state: 'pending', attemptId } + } + + const next = copyState(this.shared.state) + const rejection = this.apply(next, plan) + if (rejection !== undefined) return rejection + + const receipt = this.makeReceipt(plan) + next.operations.set(key, { semanticDigest, receipt: clone(receipt) }) + const loseReply = this.faults.loseReplyAfterCommit + this.faults.loseReplyAfterCommit = false + if (loseReply) { + const attemptId = `attempt-${next.nextAttempt++}` + next.attempts.set(key, { key: attemptId, semanticDigest, state: 'unknown' }) + this.shared.state = next + return { state: 'pending', attemptId } + } + + this.shared.state = next + return { state: 'committed', receipt: clone(receipt) } + } + + async reconcileAdmission( + key: AdmissionOperationKey, + attemptId?: string + ): Promise { + const recordKey = operationKey(key) + const attempt = this.shared.state.attempts.get(recordKey) + const saved = this.shared.state.operations.get(recordKey) + if (saved !== undefined && saved.semanticDigest !== key.semanticDigest) { + return { state: 'rejected', code: 'digest-mismatch' } + } + if (attempt === undefined) { + if (saved !== undefined) return { state: 'committed', receipt: clone(saved.receipt) } + const recoveredAttempt = this.newAttempt(recordKey, key.semanticDigest, 'unknown') + return { state: 'pending', attemptId: recoveredAttempt } + } + if (attempt.semanticDigest !== key.semanticDigest) + return { state: 'rejected', code: 'digest-mismatch' } + if (attemptId !== undefined && attempt.key !== attemptId) { + return { state: 'pending', attemptId: attempt.key } + } + if (attempt.state === 'aborted') { + this.shared.state.attempts.delete(recordKey) + return { state: 'aborted' } + } + if (saved === undefined) return { state: 'pending', attemptId: attempt.key } + this.shared.state.attempts.delete(recordKey) + return { state: 'committed', receipt: clone(saved.receipt) } + } + + private planSemanticDigest(plan: AdmissionCommit): string | undefined { + try { + return admissionSemanticDigest(plan.identity) + } catch { + return undefined + } + } + + private newAttempt(key: string, semanticDigest: string, state: Attempt['state']): string { + const attemptId = `attempt-${this.shared.state.nextAttempt++}` + this.shared.state.attempts.set(key, { key: attemptId, semanticDigest, state }) + return attemptId + } + + private apply( + next: ReferenceState, + plan: AdmissionCommit + ): Extract | undefined { + const topics = new Set(plan.identity.topics.map(item => item.topic)) + const decisionTopics = new Set(plan.decisions.map(decision => decision.topic)) + if ( + topics.size !== plan.identity.topics.length || + plan.decisions.length !== topics.size || + decisionTopics.size !== plan.decisions.length || + decisionTopics.size !== topics.size || + plan.decisions.some(decision => !topics.has(decision.topic)) || + new Set(plan.outbox.map(intent => intent.eventId)).size !== plan.outbox.length || + plan.outbox.some(intent => next.outbox.has(outboxKey(plan.identity.scope, intent.eventId))) + ) { + return { state: 'rejected', code: 'invalid-plan' } + } + if (!this.isSupportedPlan(plan, next)) return { state: 'rejected', code: 'invalid-plan' } + + const references = this.references(plan) + if ( + references === undefined || + references.some(ref => !samePayload(next.payloads.get(ref.digest), ref)) + ) { + return { state: 'rejected', code: 'payload-not-ready' } + } + + for (const decision of plan.decisions) { + if ( + !sameFence( + next.fences.get(topicKey(plan.identity.scope, decision.topic)), + decision.expectedHistory + ) + ) { + return { state: 'rejected', code: 'read-conflict' } + } + if ( + decision.reads.some( + read => + (next.reads.get(readKey(plan.identity.scope, decision.topic, read.key)) ?? null) !== + read.expectedVersion + ) + ) { + return { state: 'rejected', code: 'read-conflict' } + } + if ( + decision.spends.some( + spend => + next.outputs.get(outputKey(plan.identity.scope, decision.topic, spend.outpoint)) + ?.version !== spend.expectedVersion || + next.outputs.get(outputKey(plan.identity.scope, decision.topic, spend.outpoint)) + ?.spentBy !== undefined + ) + ) { + return { state: 'rejected', code: 'spend-conflict' } + } + if ( + decision.historyUpdate?.handoff !== undefined && + !this.canHandoff( + next, + plan.identity.scope, + decision.topic, + decision.expectedHistory, + decision.historyUpdate.handoff + ) + ) { + return { state: 'rejected', code: 'read-conflict' } + } + } + + for (const reference of references) next.pins.add(reference.digest) + for (const decision of plan.decisions) { + for (const spend of decision.spends) { + const stored = next.outputs.get( + outputKey(plan.identity.scope, decision.topic, spend.outpoint) + ) + if (stored !== undefined) stored.spentBy = spend.spender + } + for (const eviction of decision.evictions) + next.outputs.delete(outputKey(plan.identity.scope, decision.topic, eviction)) + for (const output of decision.outputs) { + const key = outputKey(plan.identity.scope, decision.topic, output) + if (next.outputs.has(key)) return { state: 'rejected', code: 'invalid-plan' } + next.outputs.set(key, { version: '1', topic: decision.topic, output: clone(output) }) + } + for (const edge of decision.edges) + next.edges.add(edgeKey(plan.identity.scope, decision.topic, edge.source, edge.consumer)) + next.applied.set( + appliedKey(plan.identity.scope, decision.topic, decision.applied.txid), + clone(decision.applied) + ) + if (decision.historyUpdate !== undefined) { + next.fences.set(topicKey(plan.identity.scope, decision.topic), { + chainEpoch: decision.expectedHistory.chainEpoch, + topicHistoryGeneration: decision.historyUpdate.nextTopicHistoryGeneration + }) + if (decision.historyUpdate.handoff !== undefined) { + const lease = next.leases.get(leaseKey(decision.historyUpdate.handoff.expected)) + if (lease !== undefined) { + lease.topicHistoryGeneration = decision.historyUpdate.nextTopicHistoryGeneration + } + next.handoffs.set( + topicKey(plan.identity.scope, decision.topic), + decision.historyUpdate.handoff.checkpoint + ) + } + next.historyUpdates.set(topicKey(plan.identity.scope, decision.topic), { + affectedFromHeight: decision.historyUpdate.affectedFromHeight, + ...(decision.historyUpdate.handoff === undefined + ? {} + : { checkpoint: decision.historyUpdate.handoff.checkpoint }) + }) + } + } + for (const intent of plan.outbox) + next.outbox.set(outboxKey(plan.identity.scope, intent.eventId), clone(intent)) + return undefined + } + + private references(plan: AdmissionCommit): AdmissionPayloadRef[] | undefined { + const references = [...plan.payloads] + for (const decision of plan.decisions) { + for (const output of decision.outputs) references.push(output.script.payload) + if (decision.applied.proof !== undefined) references.push(decision.applied.proof) + } + for (const intent of plan.outbox) references.push(...intent.payloads) + if (references.some(ref => ref.digest.length === 0 || ref.byteLength.length === 0)) + return undefined + return references + } + + private isSupportedPlan(statePlan: AdmissionCommit, state: ReferenceState): boolean { + if ( + statePlan.identity.mode === 'historical' && + statePlan.outbox.some(intent => intent.kind === 'propagation') + ) { + return false + } + const payloads = this.references(statePlan) + if ( + payloads === undefined || + payloads.some(ref => !isHash(ref.digest) || !isUint64(ref.byteLength)) + ) + return false + if (!this.isBoundSteak(statePlan)) return false + for (const decision of statePlan.decisions) { + if ( + !isUint64(decision.expectedHistory.chainEpoch) || + !isUint64(decision.expectedHistory.topicHistoryGeneration) + ) + return false + if ( + decision.spends.some( + spend => !isWireOutpoint(spend.outpoint) || spend.spender !== statePlan.identity.txid + ) + ) + return false + if ( + decision.evictions.some( + eviction => + !isWireOutpoint(eviction) || + !state.outputs.has(outputKey(statePlan.identity.scope, decision.topic, eviction)) + ) + ) + return false + if ( + decision.outputs.some(output => { + if (output.txid !== statePlan.identity.txid || !isWireOutpoint(output)) return true + if ( + ![output.satoshis, output.score, output.script.offset, output.script.byteLength].every( + isUint64 + ) + ) + return true + if (!isHash(output.script.payload.digest) || !isUint64(output.script.payload.byteLength)) + return true + return ( + parseStorageUint64(output.script.offset) + + parseStorageUint64(output.script.byteLength) > + parseStorageUint64(output.script.payload.byteLength) + ) + }) + ) + return false + if ( + decision.edges.some(edge => !isWireOutpoint(edge.source) || !isWireOutpoint(edge.consumer)) + ) + return false + const applied = decision.applied + if ( + applied.txid !== statePlan.identity.txid || + !isHash(applied.txid) || + state.applied.has(appliedKey(statePlan.identity.scope, decision.topic, applied.txid)) + ) + return false + if ( + (applied.firstSeenHeight !== undefined && !isUint64(applied.firstSeenHeight)) || + (applied.proof !== undefined && + (!isHash(applied.proof.digest) || !isUint64(applied.proof.byteLength))) || + (applied.block !== undefined && + (![applied.block.height, applied.block.index].every(isUint64) || + !isHash(applied.block.hash) || + !isHash(applied.block.merkleRoot))) + ) + return false + if (decision.historyUpdate !== undefined) { + if ( + !isUint64(decision.historyUpdate.nextTopicHistoryGeneration) || + !isUint64(decision.historyUpdate.affectedFromHeight) || + parseStorageUint64(decision.historyUpdate.nextTopicHistoryGeneration) <= + parseStorageUint64(decision.expectedHistory.topicHistoryGeneration) + ) + return false + } + } + return true + } + + private isBoundSteak(plan: AdmissionCommit): boolean { + let steak: unknown + try { + steak = JSON.parse(plan.steak) + } catch { + return false + } + if (typeof steak !== 'object' || steak === null || Array.isArray(steak)) return false + const record = steak as Record + if (Object.keys(record).length !== plan.decisions.length) return false + return plan.decisions.every(decision => { + const entry = record[decision.topic] + if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) return false + const result = entry as Record + if (!decision.outputs.every(isWireOutpoint)) return false + const admitted = decision.outputs.map(output => parseStorageOutputIndex(output.outputIndex)) + return ( + Array.isArray(result.outputsToAdmit) && + Array.isArray(result.coinsToRetain) && + Array.isArray(result.coinsRemoved) && + JSON.stringify(result.outputsToAdmit) === JSON.stringify(admitted) && + result.coinsToRetain.length === 0 && + result.coinsRemoved.length === 0 + ) + }) + } + + private canHandoff( + state: ReferenceState, + scope: StorageScope, + topic: string, + expectedHistory: HistoryFence, + handoff: HistoryRevisionHandoff + ): boolean { + const lease = handoff.expected + const current = state.leases.get(leaseKey(lease)) + return ( + current !== undefined && + sameScope(lease.scope, scope) && + lease.topic === topic && + sameFence(current, expectedHistory) && + isRecoveryLeaseCurrent(lease, current, state.nowMs) + ) + } + + private makeReceipt(plan: AdmissionCommit): AdmissionReceipt { + const lookupTargets = [ + ...new Set( + plan.outbox.filter(intent => intent.kind === 'lookup').map(intent => intent.target) + ) + ] + return { + operationId: plan.key.operationId, + semanticDigest: plan.key.semanticDigest, + durability: 'atomic-local', + steak: plan.steak, + indexes: lookupTargets.map(target => ({ target, state: 'pending' as const })), + propagation: plan.outbox.some(intent => intent.kind === 'propagation') + ? 'pending' + : 'not-requested' + } + } +} + +export class ReferenceAdmissionHarness implements AdmissionStorageContractHarness { + private readonly shared = { state: newState() } + private readonly faultState = { loseReplyAfterCommit: false, abortBeforeCommit: false } + private readonly options: ReferenceAdmissionStorageOptions + readonly seed: AdmissionStorageSeeds + readonly faults: AdmissionStorageFaults + + constructor(options: ReferenceAdmissionStorageOptions = {}) { + this.options = options + this.seed = { + readyPayload: ref => { + this.shared.state.payloads.set(ref.digest, clone(ref)) + }, + read: (scope, topic, key, version) => { + this.shared.state.reads.set(readKey(scope, topic, key), version) + }, + spendable: (scope, topic, outpoint, version) => { + this.shared.state.outputs.set(outputKey(scope, topic, outpoint), { version }) + }, + history: (scope, topic, fence) => { + this.shared.state.fences.set(topicKey(scope, topic), clone(fence)) + }, + lease: lease => { + this.shared.state.leases.set(leaseKey(lease), clone(lease)) + }, + now: milliseconds => { + this.shared.state.nowMs = milliseconds + } + } + this.faults = { + loseReplyAfterCommitOnce: () => { + this.faultState.loseReplyAfterCommit = true + }, + abortBeforeCommitOnce: () => { + this.faultState.abortBeforeCommit = true + } + } + } + + get storage(): { admission: AdmissionStorage } { + return { admission: this.restart() } + } + + restart(): AdmissionStorage { + return new ReferenceAdmissionStorage(this.shared, this.faultState, this.options) + } + + reset(): void { + this.shared.state = newState() + this.faultState.loseReplyAfterCommit = false + this.faultState.abortBeforeCommit = false + } + + snapshot(): AdmissionStorageTestSnapshot { + const state = this.shared.state + return { + payloads: [...state.payloads.values()].map(ref => ({ + ref: clone(ref), + pinned: state.pins.has(ref.digest) + })), + reads: [...state.reads].map(([key, version]) => ({ key, version })), + fences: [...state.fences].map(([key, fence]) => ({ + topic: JSON.parse(key)[1] as string, + fence: clone(fence) + })), + leases: [...state.leases.values()].map(lease => clone(lease)), + outputs: [...state.outputs].map(([key, output]) => ({ key, ...clone(output) })), + edges: [...state.edges].sort(), + applied: [...state.applied].map(([key, record]) => ({ key, record: clone(record) })), + outbox: [...state.outbox.keys()].map(key => { + const [network, genesisHash, nodeId, eventId] = JSON.parse(key) as [ + string, + string, + string, + string + ] + return { scope: { network, genesisHash, nodeId }, eventId } + }), + handoffs: [...state.handoffs].map(([key, checkpoint]) => ({ + topic: JSON.parse(key)[1] as string, + checkpoint + })), + historyUpdates: [...state.historyUpdates].map(([key, update]) => ({ + topic: JSON.parse(key)[1] as string, + ...clone(update) + })), + operations: [...state.operations].map(([key, operation]) => ({ + key, + semanticDigest: operation.semanticDigest + })) + } + } +} + +/** The shared fixture scope used by the bounded reference harness. */ +export const referenceScope: StorageScope = { + network: 'testnet', + genesisHash: '11'.repeat(32), + nodeId: 'reference-node' +} + +export const replaySafeProjection = (): ReplaySafeProjection => ({ + protocol: 'overlay-projection-v1', + applyEvent: async (_scope, _intent) => ({ checkpoint: 'checkpoint' }), + reconcile: async (_scope, _checkpoint) => ({ checkpoint: 'checkpoint' }) +}) diff --git a/packages/overlays/overlay/src/storage/AdmissionStorage.ts b/packages/overlays/overlay/src/storage/AdmissionStorage.ts new file mode 100644 index 000000000..6d861fd8f --- /dev/null +++ b/packages/overlays/overlay/src/storage/AdmissionStorage.ts @@ -0,0 +1,254 @@ +import { createHash } from 'node:crypto' +import type { HistoryRevisionHandoff } from './RecoveryContract.js' + +/** Canonical unsigned decimal, 0..2^64-1. Never round through a JS number. */ +export type StorageUint64 = string + +export interface StorageScope { + network: string + genesisHash: string + nodeId: string +} + +/** These revisions are independent: a late admission need not be a reorg. */ +export interface HistoryFence { + chainEpoch: StorageUint64 + topicHistoryGeneration: StorageUint64 +} + +export interface AdmissionIdentity { + scope: StorageScope + /** Derived from verified immutable transaction bytes before storage is called. */ + txid: string + mode: 'live' | 'historical' + /** Digest of off-chain values and any other immutable admission-policy inputs. */ + contextDigest: string + topics: Array<{ topic: string; policyId: string }> +} + +export interface AdmissionOperationKey { + scope: StorageScope + operationId: string + semanticDigest: string +} + +/** Small reference to content published outside the database transaction. */ +export interface AdmissionPayloadRef { + digest: string + byteLength: StorageUint64 + kind: 'raw-transaction' | 'merkle-path' | 'beef-manifest' | 'locking-script' | 'outbox-data' +} + +export interface AdmissionOutpoint { + txid: string + /** Canonical decimal uint32, the transaction wire outpoint index domain. */ + outputIndex: StorageUint64 +} + +/** Adapter-issued revision token; absence is also a read predicate. */ +export interface AdmissionReadPredicate { + key: string + expectedVersion: string | null +} + +export interface AdmissionOutput extends AdmissionOutpoint { + satoshis: StorageUint64 + score: StorageUint64 + /** A byte range supports even an individually oversized output script. */ + script: { payload: AdmissionPayloadRef; offset: StorageUint64; byteLength: StorageUint64 } +} + +export interface AdmissionTopicDecision { + topic: string + expectedHistory: HistoryFence + /** Includes all point/range/projection reads that influenced topic admission. */ + reads: AdmissionReadPredicate[] + spends: Array<{ outpoint: AdmissionOutpoint; expectedVersion: string; spender: string }> + /** Remove current serving outputs only; retain applied history and evidence. */ + evictions: AdmissionOutpoint[] + outputs: AdmissionOutput[] + edges: Array<{ source: AdmissionOutpoint; consumer: AdmissionOutpoint }> + /** Retain this history after output spend, serving moderation or index eviction. */ + applied: { + txid: string + firstSeenHeight?: StorageUint64 + proof?: AdmissionPayloadRef + block?: { height: StorageUint64; hash: string; index: StorageUint64; merkleRoot: string } + } + /** Invalidate downstream anchors/comparisons; own repair checkpoint moves atomically. */ + historyUpdate?: { + nextTopicHistoryGeneration: StorageUint64 + affectedFromHeight: StorageUint64 + /** Present when a recovery worker itself caused this history revision. */ + handoff?: HistoryRevisionHandoff + } +} + +/** Every event has a stable scope-local ID and ready payload references. */ +export interface AdmissionOutboxIntent { + eventId: string + kind: 'lookup' | 'propagation' + target: string + payloads: AdmissionPayloadRef[] +} + +export interface AdmissionCommit { + key: AdmissionOperationKey + identity: AdmissionIdentity + /** Shared raw/proof components or a manifest; never per-output inline BEEF. */ + payloads: AdmissionPayloadRef[] + decisions: AdmissionTopicDecision[] + outbox: AdmissionOutboxIntent[] + /** Exact UTF-8 STEAK JSON saved with the commit, then replayed without regeneration. */ + steak: string +} + +export interface AdmissionReceipt { + operationId: string + semanticDigest: string + /** Only the selected adapter can establish this guarantee. */ + durability: 'atomic-local' + steak: string + /** Only enlisted indexes may be visible at commit. External indexes remain pending. */ + indexes: Array<{ target: string; state: 'visible' | 'pending' }> + propagation: 'not-requested' | 'pending' +} + +/** Pending is not an ACK. Unknown commit must retain its opaque attempt identity. */ +export type AdmissionCommitResult = + | { state: 'committed'; receipt: AdmissionReceipt } + | { state: 'pending'; attemptId: string } + | { + state: 'rejected' + code: + | 'digest-mismatch' + | 'read-conflict' + | 'spend-conflict' + | 'payload-not-ready' + | 'unsupported-projection' + | 'invalid-plan' + } + +export type AdmissionReconcileResult = AdmissionCommitResult | { state: 'aborted' } + +/** + * Optional v1 contract, separate from legacy CRUD Storage. No current adapter or + * Engine call path implements it. A provider must atomically revalidate reads, + * fences, conditional spends and ready-payload pins, and save every effect plus + * the receipt. Matching retries return the saved receipt before checking stale + * reads; conflicting semantic digests reject. Do not run verification, uploads, + * network requests or arbitrary plug-ins inside commitAdmission. + * + * TransientTransactionError permits a fresh whole-body attempt after abort and + * reread/redecision. UnknownTransactionCommitResult requires reconcileAdmission + * on the SAME attempt until committed or definitively aborted; absence of an + * operation record alone does not establish abort. Repeated commitAdmission + * must first recover any persisted unresolved attempt for this operation, even + * after a lost response or process restart, before it can start a fresh body. + * Transport failures remain + * errors, never rejection/abort or a successful local ACK. + */ +export interface AdmissionStorage { + readonly protocol: 'overlay-admission-v1' + commitAdmission: (plan: AdmissionCommit) => Promise + reconcileAdmission: ( + key: AdmissionOperationKey, + attemptId?: string + ) => Promise +} + +/** External projection delivery may be retried only with both capabilities. */ +export interface ReplaySafeProjection { + readonly protocol: 'overlay-projection-v1' + applyEvent: ( + scope: StorageScope, + intent: AdmissionOutboxIntent + ) => Promise<{ checkpoint: string }> + reconcile: (scope: StorageScope, checkpoint: string | null) => Promise<{ checkpoint: string }> +} + +/** Shape detection is a declaration, not proof that an adapter is durable. */ +export function getAdmissionStorage(storage: unknown): AdmissionStorage | undefined { + if (typeof storage !== 'object' || storage === null || !('admission' in storage)) return undefined + const candidate = storage.admission + if (typeof candidate !== 'object' || candidate === null) return undefined + if (!('protocol' in candidate) || candidate.protocol !== 'overlay-admission-v1') return undefined + if (!('commitAdmission' in candidate) || typeof candidate.commitAdmission !== 'function') + return undefined + if (!('reconcileAdmission' in candidate) || typeof candidate.reconcileAdmission !== 'function') + return undefined + return candidate as AdmissionStorage +} + +export function isReplaySafeProjection(projection: unknown): projection is ReplaySafeProjection { + return ( + typeof projection === 'object' && + projection !== null && + 'protocol' in projection && + projection.protocol === 'overlay-projection-v1' && + 'applyEvent' in projection && + typeof projection.applyEvent === 'function' && + 'reconcile' in projection && + typeof projection.reconcile === 'function' + ) +} + +export function parseStorageUint64(value: string): bigint { + if (typeof value !== 'string' || !/^(0|[1-9][0-9]{0,19})$/.test(value)) + throw new Error('Invalid storage uint64') + const result = BigInt(value) + if (result > BigInt('18446744073709551615')) throw new Error('Invalid storage uint64') + return result +} + +/** Validate the wire outpoint domain before conversion to an exact STEAK number. */ +export function parseStorageOutputIndex(value: string): number { + const result = parseStorageUint64(value) + if (result > BigInt('4294967295')) throw new Error('Invalid storage output index') + return Number(result) +} + +/** + * SHA-256 of UTF-8 length-framed fields, see specs/overlay/persistence-v1.md. + * Mutable reads/decisions and proof variants are deliberately outside semantic + * identity. Their validation and atomic read predicates remain mandatory. + */ +export function admissionSemanticDigest(identity: AdmissionIdentity): string { + if ( + !/^[0-9a-f]{64}$/.test(identity.txid) || + !/^[0-9a-f]{64}$/.test(identity.scope.genesisHash) || + !/^[0-9a-f]{64}$/.test(identity.contextDigest) + ) { + throw new Error('Invalid admission hash') + } + if (identity.mode !== 'live' && identity.mode !== 'historical') + throw new Error('Invalid admission mode') + const topics = [...identity.topics].sort((a, b) => + Buffer.compare(Buffer.from(a.topic), Buffer.from(b.topic)) + ) + if ( + topics.length === 0 || + topics.some((item, index) => index > 0 && item.topic === topics[index - 1].topic) + ) { + throw new Error('Invalid admission topics') + } + const fields = [ + 'overlay-admission-v1', + identity.scope.network, + identity.scope.genesisHash, + identity.scope.nodeId, + identity.txid, + identity.mode, + identity.contextDigest, + String(topics.length), + ...topics.flatMap(({ topic, policyId }) => [topic, policyId]) + ] + const hash = createHash('sha256') + for (const field of fields) { + if (field.length === 0 || !field.isWellFormed()) throw new Error('Invalid admission identity') + const bytes = Buffer.from(field, 'utf8') + hash.update(String(bytes.length) + ':') + hash.update(bytes) + } + return hash.digest('hex') +} diff --git a/packages/overlays/overlay/src/storage/RecoveryContract.ts b/packages/overlays/overlay/src/storage/RecoveryContract.ts new file mode 100644 index 000000000..289d3a57f --- /dev/null +++ b/packages/overlays/overlay/src/storage/RecoveryContract.ts @@ -0,0 +1,87 @@ +import { + parseStorageUint64, + type HistoryFence, + type StorageScope, + type StorageUint64 +} from './AdmissionStorage.js' + +export interface RecoveryLease extends HistoryFence { + scope: StorageScope + topic: string + peerId: string + jobId: string + leaseToken: StorageUint64 + expiresAtMs: StorageUint64 +} + +/** Use database time and this predicate INSIDE the checkpoint/publication CAS. */ +export function isRecoveryLeaseCurrent( + expected: RecoveryLease, + current: RecoveryLease, + nowMs: StorageUint64 +): boolean { + return ( + expected.scope.network === current.scope.network && + expected.scope.genesisHash === current.scope.genesisHash && + expected.scope.nodeId === current.scope.nodeId && + expected.topic === current.topic && + expected.peerId === current.peerId && + expected.jobId === current.jobId && + parseStorageUint64(expected.chainEpoch) === parseStorageUint64(current.chainEpoch) && + parseStorageUint64(expected.topicHistoryGeneration) === + parseStorageUint64(current.topicHistoryGeneration) && + parseStorageUint64(expected.leaseToken) === parseStorageUint64(current.leaseToken) && + parseStorageUint64(current.expiresAtMs) > parseStorageUint64(nowMs) + ) +} + +/** A repair's own revision/checkpoint must move together; other workers rewind. */ +export interface HistoryRevisionHandoff { + expected: RecoveryLease + checkpoint: string +} + +/** Immutable history; a current pointer must CAS both revisions and the header. */ +export interface TopicAnchorRevision extends HistoryFence { + scope: StorageScope + topic: string + height: StorageUint64 + blockHash: string + basmRoot: string + admittedCount: StorageUint64 + tac: string + previousTac: string + policyId: string +} + +/** A local tuple does not negotiate a remote protocol. Evidence comes from S06. */ +export type GaspCursorEvidence = + | { mode: 'negotiated-tuple'; negotiated: boolean; pageFinalized: boolean } + | { + mode: 'inclusive-since' + inclusiveSemanticsProven: boolean + equalScoreDrained: boolean + pageFinalized: boolean + } + | { + mode: 'full-resync' + noSkipSemanticsProven: boolean + resyncCompleted: boolean + pageFinalized: boolean + } + | { mode: 'unsupported'; pageFinalized: boolean } + +/** Gate for a durable finalization/cursor transaction; never advances a cursor. */ +export function canAdvanceGaspCursor(evidence: GaspCursorEvidence): boolean { + if (!evidence.pageFinalized) return false + switch (evidence.mode) { + case 'negotiated-tuple': + return evidence.negotiated + case 'inclusive-since': + return evidence.inclusiveSemanticsProven && evidence.equalScoreDrained + case 'full-resync': + return evidence.noSkipSemanticsProven && evidence.resyncCompleted + default: + return false + } +} diff --git a/packages/overlays/overlay/src/storage/Storage.ts b/packages/overlays/overlay/src/storage/Storage.ts index ebcfee9bb..80b08be8a 100644 --- a/packages/overlays/overlay/src/storage/Storage.ts +++ b/packages/overlays/overlay/src/storage/Storage.ts @@ -1,4 +1,5 @@ import type { Output } from '../Output.js' +import type { AdmissionStorage } from './AdmissionStorage.js' import type { AdmittedTxRef, RawTransactionRecord, @@ -59,6 +60,9 @@ export interface UnprovenAppliedTransactionCandidate { * Defines the Storage Engine interface used internally by the Overlay Services Engine. */ export interface Storage { + /** Optional whole-admission contract. Legacy CRUD does not imply this capability. */ + readonly admission?: AdmissionStorage + /** * Adds a new output to storage * @param utxo — The output to add diff --git a/packages/overlays/topics/CHANGELOG.md b/packages/overlays/topics/CHANGELOG.md index b47721adb..a08e7979f 100644 --- a/packages/overlays/topics/CHANGELOG.md +++ b/packages/overlays/topics/CHANGELOG.md @@ -9,6 +9,8 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +- Updates the packed workspace dependency candidate for the additive overlay persistence contract. Runtime behavior and defaults are unchanged; no consumer migration is required. + ### Added - `tm_uora_dpp` / `ls_uora_dpp`: admission and lookup for UORA attestation diff --git a/packages/overlays/topics/package.json b/packages/overlays/topics/package.json index 0b202c4d8..1f1c0f0ad 100644 --- a/packages/overlays/topics/package.json +++ b/packages/overlays/topics/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-topics", - "version": "1.7.1", + "version": "1.7.2", "sideEffects": false, "engines": { "node": ">=22" diff --git a/specs/overlay/fixtures/persistence-v1.json b/specs/overlay/fixtures/persistence-v1.json new file mode 100644 index 000000000..37958de48 --- /dev/null +++ b/specs/overlay/fixtures/persistence-v1.json @@ -0,0 +1,536 @@ +{ + "version": 1, + "provenance": "Synthetic S01 logical contract; SHA256 expectations calculated independently with Python hashlib and UTF-8 byte framing. No database durability or protocol activation evidence.", + "identities": [ + { + "name": "unsorted-topics", + "identity": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "txid": "1111111111111111111111111111111111111111111111111111111111111111", + "mode": "live", + "contextDigest": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "topics": [ + { + "topic": "tm_z", + "policyId": "v1" + }, + { + "topic": "tm_a", + "policyId": "v2" + } + ] + }, + "digest": "c5f5a8fd3e13865abb215aca4e43b6a55b08573d5448aecc7d50111c0b8d3688" + }, + { + "name": "historical-mode", + "identity": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "txid": "1111111111111111111111111111111111111111111111111111111111111111", + "mode": "historical", + "contextDigest": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "topics": [ + { + "topic": "tm_z", + "policyId": "v1" + }, + { + "topic": "tm_a", + "policyId": "v2" + } + ] + }, + "digest": "71ac68769556e6b6ca885588f7f6533bb716e9b98ce6dad59e1618174937958a" + }, + { + "name": "utf8-order", + "identity": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "txid": "1111111111111111111111111111111111111111111111111111111111111111", + "mode": "live", + "contextDigest": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "topics": [ + { + "topic": "𐀀", + "policyId": "policy:1" + }, + { + "topic": "", + "policyId": "policy:2" + } + ] + }, + "digest": "8a8dbf7a4a5422a15f4e36856a1feda982ae65b6ad92dd0aaecd753e157ebd98" + }, + { + "name": "framed-delimiters", + "identity": { + "scope": { + "network": "test:net", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "n:1" + }, + "txid": "1111111111111111111111111111111111111111111111111111111111111111", + "mode": "live", + "contextDigest": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "topics": [ + { + "topic": "tm:1", + "policyId": "a:b" + } + ] + }, + "digest": "38a2ac07d4e5108c0c60fa37d682580aa6a8be865b8f84347c64b8fc2b734833" + } + ], + "uint64": [ + { + "value": "0", + "valid": true + }, + { + "value": "1", + "valid": true + }, + { + "value": "9007199254740993", + "valid": true + }, + { + "value": "18446744073709551615", + "valid": true + }, + { + "value": "18446744073709551616", + "valid": false + }, + { + "value": "01", + "valid": false + }, + { + "value": "-1", + "valid": false + }, + { + "value": "+1", + "valid": false + }, + { + "value": "1.0", + "valid": false + }, + { + "value": "1e3", + "valid": false + }, + { + "value": "", + "valid": false + }, + { + "value": " 1", + "valid": false + } + ], + "cursors": [ + { + "name": "negotiated-tuple", + "evidence": { + "mode": "negotiated-tuple", + "pageFinalized": true, + "negotiated": true + }, + "advance": true + }, + { + "name": "negotiated-tuple-uncommitted", + "evidence": { + "mode": "negotiated-tuple", + "pageFinalized": false, + "negotiated": true + }, + "advance": false + }, + { + "name": "negotiated-tuple-negotiated-false", + "evidence": { + "mode": "negotiated-tuple", + "pageFinalized": true, + "negotiated": false + }, + "advance": false + }, + { + "name": "inclusive-since", + "evidence": { + "mode": "inclusive-since", + "pageFinalized": true, + "inclusiveSemanticsProven": true, + "equalScoreDrained": true + }, + "advance": true + }, + { + "name": "inclusive-since-uncommitted", + "evidence": { + "mode": "inclusive-since", + "pageFinalized": false, + "inclusiveSemanticsProven": true, + "equalScoreDrained": true + }, + "advance": false + }, + { + "name": "inclusive-since-inclusiveSemanticsProven-false", + "evidence": { + "mode": "inclusive-since", + "pageFinalized": true, + "inclusiveSemanticsProven": false, + "equalScoreDrained": true + }, + "advance": false + }, + { + "name": "inclusive-since-equalScoreDrained-false", + "evidence": { + "mode": "inclusive-since", + "pageFinalized": true, + "inclusiveSemanticsProven": true, + "equalScoreDrained": false + }, + "advance": false + }, + { + "name": "full-resync", + "evidence": { + "mode": "full-resync", + "pageFinalized": true, + "noSkipSemanticsProven": true, + "resyncCompleted": true + }, + "advance": true + }, + { + "name": "full-resync-uncommitted", + "evidence": { + "mode": "full-resync", + "pageFinalized": false, + "noSkipSemanticsProven": true, + "resyncCompleted": true + }, + "advance": false + }, + { + "name": "full-resync-noSkipSemanticsProven-false", + "evidence": { + "mode": "full-resync", + "pageFinalized": true, + "noSkipSemanticsProven": false, + "resyncCompleted": true + }, + "advance": false + }, + { + "name": "full-resync-resyncCompleted-false", + "evidence": { + "mode": "full-resync", + "pageFinalized": true, + "noSkipSemanticsProven": true, + "resyncCompleted": false + }, + "advance": false + }, + { + "name": "unsupported", + "evidence": { + "mode": "unsupported", + "pageFinalized": true + }, + "advance": false + }, + { + "name": "unsupported-uncommitted", + "evidence": { + "mode": "unsupported", + "pageFinalized": false + }, + "advance": false + } + ], + "leases": [ + { + "name": "current", + "expected": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "current": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "nowMs": "999", + "valid": true + }, + { + "name": "expired", + "expected": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "current": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "nowMs": "1000", + "valid": false + }, + { + "name": "chain-reorg", + "expected": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "current": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740994", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "nowMs": "999", + "valid": false + }, + { + "name": "same-block-history", + "expected": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "current": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "6", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "nowMs": "999", + "valid": false + }, + { + "name": "replaced-worker", + "expected": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "current": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "8", + "expiresAtMs": "1000" + }, + "nowMs": "999", + "valid": false + }, + { + "name": "different-peer", + "expected": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "current": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-b", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "nowMs": "999", + "valid": false + }, + { + "name": "different-node", + "expected": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-a" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "current": { + "scope": { + "network": "test", + "genesisHash": "0000000000000000000000000000000000000000000000000000000000000000", + "nodeId": "node-b" + }, + "topic": "tm_a", + "peerId": "peer-a", + "jobId": "job-a", + "chainEpoch": "9007199254740993", + "topicHistoryGeneration": "5", + "leaseToken": "7", + "expiresAtMs": "1000" + }, + "nowMs": "999", + "valid": false + } + ], + "outputIndices": [ + { + "value": "0", + "valid": true + }, + { + "value": "4294967295", + "valid": true + }, + { + "value": "4294967296", + "valid": false + }, + { + "value": "9007199254740992", + "valid": false + }, + { + "value": "9007199254740993", + "valid": false + }, + { + "value": "01", + "valid": false + }, + { + "value": "-1", + "valid": false + } + ] +} diff --git a/specs/overlay/persistence-v1.md b/specs/overlay/persistence-v1.md new file mode 100644 index 000000000..aa4bae016 --- /dev/null +++ b/specs/overlay/persistence-v1.md @@ -0,0 +1,163 @@ +# Overlay persistence contract v1 + +W02 persistence / S01; requirements S1, S2, S3, B2, B3 and X1. This is an +additive logical adapter contract and executable reference seam. It is not a +Mongo schema, a new public HTTP shape, or authorization for TS and Go writers to +share a node database. No production adapter implements this capability yet. + +## Baseline and selection + +At TS base `2bc799a8d8e535242e6de2d305f426ce3975ea7b`, `Engine.submit` broadcasts +and invokes its STEAK callback before persistence. Mutations and lookup callbacks +are separate operations; some errors are logged without rejecting the submission. +Knex transactions inside individual methods do not make that sequence atomic. +Go base `f5d8074be085c1970e113bc8ee55fecbab251fcf` likewise injects `engine.Storage` +and performs separate commit/lookup writes. Its GASP graph interface separates +append, validate, discard and finalize without a durable admission/cursor boundary. + +TS `Storage.admission` optionally declares `overlay-admission-v1` and provides +`commitAdmission` and `reconcileAdmission`. `getAdmissionStorage` checks those +declarations only. Go uses a separate optional provider/interface, preserving its +existing `engine.Storage` implementers. Shape detection cannot certify durability. +The existing engines, SQL adapter, injected Go stores and default selections are +unchanged. Their existing STEAK callback must not be described as this receipt. +S03/S04 must integrate and verify the capability before activating durable ACKs. + +## Identity and exact values + +Keys are tuples, never ambiguous concatenation: `(network, genesisHash, nodeId, +operationId)`. Topic, peer and job keys add their respective identities. Every +hash is 32 bytes represented as 64 lowercase hex characters. Content bytes are +binary; JSON fixtures use hex. New logical unsigned values are canonical decimal +strings `0|[1-9][0-9]*`, bounded by `18446744073709551615`. This includes satoshis, +scores, offsets, lengths, heights, output/block indices, epochs and lease tokens. +Existing numeric APIs/wire formats are unchanged. Adapters must impose the +additional transaction/protocol bounds and never round via a JavaScript number. +`parseStorageOutputIndex` validates canonical decimal uint32 (`0..4294967295`) +before producing an exact STEAK number; Go's `ParseStorageOutputIndex` returns +the equivalent `uint32`. The general uint64 parser must not replace this bound. + +`admissionSemanticDigest` is SHA-256 of the following UTF-8 fields in order: + +1. `overlay-admission-v1`, network, genesisHash, nodeId, verified raw txid, mode, + contextDigest and decimal topic count; +2. each `(topic, policyId)`, sorted by topic's unsigned UTF-8 bytes. + +Each field is framed as ASCII decimal UTF-8 byte length, `:`, then its bytes. +There is no separator between framed fields, no BOM and no Unicode normalization. +Reject empty fields, ill-formed Unicode, duplicate topics, invalid hashes and +unknown modes. The modes are `live` and `historical`; future engine adapters must +explicitly map their verification/replay policy. `contextDigest` binds off-chain +values and any other immutable admission-policy inputs; their caller-defined +canonical encoding belongs to the named policy. SHA-256 of empty bytes represents +an empty context. Changing those inputs or policy changes identity. + +Proof variants, temporary read versions, scores and recalculated decisions do +not enter semantic identity. All raw txids must already be derived and verified. +A later valid proof cannot falsely conflict with an admitted raw transaction. +Matching retries return the original saved receipt; a separate validated proof +update remains required to persist better evidence. This slice does not implement +that update. Different operation IDs still cannot duplicate applied/outpoint keys. + +## Atomic admission and observable outcome + +An adapter validates the bounded plan before entering a transaction. It must +recompute the digest, bind scope/topics/txids and the saved STEAK to the decision, +check exact integer/range formats, and reject unsupported plan effects. Reads +include absent records and range/projection revisions: an untracked external read +cannot silently participate in optimistic admission. Recompute a stale topic +decision outside the transaction under the same logical operation identity. + +One commit checks operation uniqueness/digest and all read predicates, both +history fences, conditional spends and payload readiness; applies outputs, +serving evictions, consumption edges and retained applied history; pins payloads; +persists history invalidation/handoff and outbox intents; and saves the exact +UTF-8 STEAK JSON plus its receipt. Existing applied/outpoint identities may be +reused only with equivalent effects, never overwritten by a competing operation. +Failure publishes none of these effects. Matching completed retry is checked +before stale reads or newer history fences and returns the saved receipt. + +The receipt's `durability: atomic-local` covers only this local commit. Index +visibility is per target: only an enlisted index can be `visible`; external +indexes are `pending` until their checkpoint proves delivery. Propagation is +`pending` or `not-requested`, never delivered merely because admission committed. +Historical admissions have no propagation intents. Retry receipts preserve the +original observation even if delivery later advances; current delivery status is +a separate read. Pending/unknown results are not ACKs. + +No verification, topic callback, network request, plug-in execution or payload +upload runs inside the database transaction. External projections require an +explicit `overlay-projection-v1` idempotent `applyEvent` plus rebuild/reconciliation +`reconcile` contract. Stable event IDs deduplicate repeats; checkpoints acknowledge +completed work. Both methods receive the storage scope explicitly; deduplication +and checkpoint identity include that scope, even when one projection serves +several nodes or networks. An ordinary lookup callback is unsupported for replay. Native +indexes may enlist only through an adapter-specific transaction mechanism. + +## Retry and payload lifecycle + +`TransientTransactionError` permits the complete body to run again only after the +previous attempt aborts, with reread/redecision where needed. +`UnknownTransactionCommitResult` retains the same opaque attempt/commit identity +and calls `reconcileAdmission(key, attemptId)` until committed or definitively +aborted. If the entire response was lost, `reconcileAdmission(key)` locates the +persisted unresolved attempt by operation key. Every repeated `commitAdmission` +must also recover that attempt before starting another body, including after a +restart. Missing operation/attempt data alone is not proof of abort; an unknown +attempt remains pending, and a known conflicting digest is rejected. A reconciliation +error remains unresolved; no fresh body, plugin replay or successful ACK follows +from a timeout. Adapter exceptions are storage/transport failures, distinct from +the bounded semantic rejection codes. Telemetry uses codes and counts, never raw +payloads, credentials, policy input or unredacted connection errors. + +Payload references carry kind, digest and exact length. A manifest refers to +shared per-transaction/proof components so overlapping BEEF ancestry is shared. +Output scripts can use ranges of published payloads, including a raw transaction +larger than a BSON document. Stage/upload/verify/publish bytes outside admission; +the short commit references only `ready` metadata and atomically pins it. GC's +`ready -> deleting` claim must serialize with every pin/reference creator and +forbid new references after the claim. Physical cleanup is recoverable separately. +This slice implements no upload, GridFS, collection, migration or garbage collector. + +## Recovery and publication + +`chainEpoch` advances on reorg. `topicHistoryGeneration` advances on a change to +historical admissions, including the same height and block hash. Immutable anchor +revisions retain both dimensions; current pointers require both and the pinned +canonical header. History updates invalidate affected anchors and downstream +comparisons before a current tip or green status can be reused. Serving bans, +spent outputs and index eviction do not remove canonical admission evidence. + +An admission that changes history supplies `historyUpdate`. Its next generation +must exceed the expected one without changing chainEpoch. If its own recovery +worker caused the revision, the same transaction checks the job's current scope, +topic, peer, job ID, both generations, unexpired database-time lease and token, +then moves that checkpoint/lease to the new history generation. Competing changes +rewind affected work. `isRecoveryLeaseCurrent` is a pure predicate to evaluate +inside that CAS; evaluating it before a write does not establish fencing. + +`canAdvanceGaspCursor` only gates proposed cursor publication. Durable graph +finalization/admission and advancement must share the transaction. A tuple is +usable only after negotiated wire support. Inclusive numeric-since requires +proven inclusive semantics and complete equal-score drain. Otherwise use a proven +completed no-skip full resync or return capability-limited with no advancement. +A finite page, strict-since peer or unknown semantics never proves ties drained. +These helpers do not change existing peers or implement the S06 scheduler. + +## Evidence and continuation + +[`fixtures/persistence-v1.json`](fixtures/persistence-v1.json) contains independent +Python hashlib expectations for UTF-8 framed identity, exact integers, cursor +conditions and lease/revision fences. TS and Go consume byte-identical copies. +The test-only admission harness exercises saved ACK replay, conflicts, atomic +reference publication and pending/reconciliation outcomes; future adapters can +run the same behavioral scenarios. An in-memory model passing them does not +establish crash durability, Mongo transaction retries or managed-service recovery. + +S02 owns database payload/schema/retry implementation; S03/S04 own engine commit +integration; S05 owns projection workers; S06 owns graph cursor negotiation and +finalization; B02/B04 own durable anchor/job/comparison storage and recovery. +Replica-set crashes, unknown commit labels, index enlistment, payload GC races, +all-equal-score peer exchange and deployment/migration evidence remain required. +No consumer migration is required for this additive S01 API. Mixed-version +concurrent TS/Go writers remain unsupported. From 7058592f1645e2095ad475041f4c62b71c888b13 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 13:37:24 -0500 Subject: [PATCH 003/168] fix(sdk): preserve safe integer Merkle path offsets --- docs/packages/sdk/bsv-sdk.md | 8 +- docs/reference/package-api-migrations.md | 6 +- docs/reference/stack-facts.md | 2 +- governance/package-release-notes.json | 6 +- packages/sdk/CHANGELOG.md | 6 + packages/sdk/README.md | 5 + packages/sdk/package.json | 2 +- packages/sdk/src/transaction/MerklePath.ts | 99 ++++---- .../__tests/MerklePath.safeOffsets.test.ts | 234 ++++++++++++++++++ 9 files changed, 314 insertions(+), 54 deletions(-) create mode 100644 packages/sdk/src/transaction/__tests/MerklePath.safeOffsets.test.ts diff --git a/docs/packages/sdk/bsv-sdk.md b/docs/packages/sdk/bsv-sdk.md index 14ef20ca4..1dee01d5f 100644 --- a/docs/packages/sdk/bsv-sdk.md +++ b/docs/packages/sdk/bsv-sdk.md @@ -3,7 +3,7 @@ id: bsv-sdk title: '@bsv/sdk' kind: package domain: sdk -version: '2.4.2' +version: '2.5.0' npm: '@bsv/sdk' last_updated: '2026-08-26' last_verified: '2026-08-26' @@ -15,6 +15,12 @@ repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk' # @bsv/sdk +The unpublished 2.5.0 candidate corrects BUMP offset arithmetic above 32 bits +through `Number.MAX_SAFE_INTEGER`, preserving existing wire encodings. Root +calculation, extraction, combination and trimming use the same exact numeric +domain; malformed non-integer and unsafe offsets fail explicitly. No consumer +migration is needed for this correction. + The foundational cryptographic and transaction library for the BSV blockchain. Zero external dependencies — all cryptographic primitives have been validated by a third-party auditor. Every other library in the ts-stack builds on top of `@bsv/sdk`. Provides low-level primitives (keys, signatures, hashing), script construction and execution, transaction creation and signing, and integration interfaces for wallets and overlay networks. diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 995d1e809..6f775f6ee 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -48,7 +48,7 @@ and clean-consumer tests remain the executable type authority. | `@bsv/overlay-topics` | `1.6.10` | `1.7.1` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. | | `@bsv/paymail` | `2.4.2` | `2.4.7` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. Consumers of the former bundled Money Button or Tokenized specification documents must follow the authoritative links in docs/specs/README.md. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/payment-express-middleware` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/sdk` | `2.4.0` | `2.4.2` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. | +| `@bsv/sdk` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/sdk/bsv-sdk.md) | No API or valid BUMP wire migration is required for the offset correction. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. | | `@bsv/simple` | `0.4.1` | `0.5.3` | minor | [API and usage](../packages/helpers/simple.md) | Existing overlay configurations and number-array behavior are unchanged. TTN consumers select network teratestnet; all consumers should upgrade to @bsv/sdk 2.4.2 or later. The Message Box Client dependency advance requires no helper API migration and preserves the default HTTP and live-socket behavior. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/. | | `@bsv/templates` | `1.9.1` | `1.10.1` | minor | [API and usage](../packages/helpers/templates.md) | No existing consumer migration is required; existing template APIs and generated scripts are unchanged. New R1K1Wallet consumers await lock(), retain each private 32-byte salt, and provide a PIV signer that signs the supplied digest directly without hashing it again. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/teranode-listener` | `1.1.1` | `1.1.5` | patch | [API and usage](../packages/network/teranode-listener.md) | No consumer migration is required; listener APIs, topics, and network configuration are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | @@ -359,8 +359,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/sdk/bsv-sdk.md](../packages/sdk/bsv-sdk.md) - Source: [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) -- Release note: Adds one strict BRC-100 byte-boundary contract and secure React Native bridge validation, and completes incorporated-code compliance with hash-pinned provenance, exact MIT/ISC/BSD/Apache notices, source lineage markers, package payloads, a retained UMD banner, SBOM components, and release enforcement. Removes a broken copied Swagger UI scaffold and screenshot; the first-party OpenAPI source remains available for use with any viewer. Standardizes first-party author metadata on the current BSV Association name. -- Migration: No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. +- Release note: The coordinated unpublished 2.5.0 candidate corrects Merkle-path arithmetic for the full safe-integer offset domain, including positions above 32 bits, and rejects nonrepresentable offsets. Retains the strict BRC-100 byte-boundary contract, secure React Native bridge validation, incorporated-code provenance and notices, source lineage markers, package payloads, UMD banner, SBOM components, release enforcement, Swagger scaffold removal, and first-party author metadata corrections from the prior candidate. +- Migration: No API or valid BUMP wire migration is required for the offset correction. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 13c442ae0..5e6e51522 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -62,7 +62,7 @@ authorized release action. | overlays | `@bsv/overlay-discovery-services` | `2.2.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | | overlays | `@bsv/overlay-express` | `2.6.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | | overlays | `@bsv/overlay-topics` | `1.7.1` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | -| sdk | `@bsv/sdk` | `2.4.2` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | +| sdk | `@bsv/sdk` | `2.5.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | | sdk | `@bsv/verifast` | `0.3.5` | wasm-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global, wasm-worker | browser, node, umd, wasm, worker | `>=22` | [packages/verifast](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/verifast) | | wallet | `@bsv/btms` | `1.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | | wallet | `@bsv/btms-permission-module` | `1.1.4` | node-library | node-esm | node | `>=22` | [packages/wallet/btms-permission-module](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms-permission-module) | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index fb190dd4d..65bd5ae07 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -167,9 +167,9 @@ { "name": "@bsv/sdk", "publishedVersion": "2.4.0", - "releaseType": "patch", - "summary": "Adds one strict BRC-100 byte-boundary contract and secure React Native bridge validation, and completes incorporated-code compliance with hash-pinned provenance, exact MIT/ISC/BSD/Apache notices, source lineage markers, package payloads, a retained UMD banner, SBOM components, and release enforcement. Removes a broken copied Swagger UI scaffold and screenshot; the first-party OpenAPI source remains available for use with any viewer. Standardizes first-party author metadata on the current BSV Association name.", - "migration": "No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles." + "releaseType": "minor", + "summary": "The coordinated unpublished 2.5.0 candidate corrects Merkle-path arithmetic for the full safe-integer offset domain, including positions above 32 bits, and rejects nonrepresentable offsets. Retains the strict BRC-100 byte-boundary contract, secure React Native bridge validation, incorporated-code provenance and notices, source lineage markers, package payloads, UMD banner, SBOM components, release enforcement, Swagger scaffold removal, and first-party author metadata corrections from the prior candidate.", + "migration": "No API or valid BUMP wire migration is required for the offset correction. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles." }, { "name": "@bsv/simple", diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index d6b61d775..8d6d6bf48 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -214,6 +214,12 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +- Corrected Merkle-path parent, sibling, cache, effective-depth, extraction and + trimming arithmetic for safe-integer transaction positions above 32 bits. + Unsafe, fractional, negative and non-finite offsets are rejected explicitly. + No valid BUMP wire change or consumer migration is required. This correction + joins the coordinated, unpublished 2.5.0 candidate. + ### Added - Add shared BRC-100 byte-boundary helpers that preserve valid `number[]` and diff --git a/packages/sdk/README.md b/packages/sdk/README.md index ee0e7d30e..3398ec730 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -85,6 +85,11 @@ For a more detailed tutorial and advanced examples, check our [Documentation](#d - **Merkle Proof Verification**: Tools for representing and verifying merkle proofs, adhering to various serialization standards. + BUMP transaction offsets retain their exact nonnegative safe-integer domain, + including positions above 32 bits. Root calculation, proof extraction, + combination, and trimming use the same full-width arithmetic. Offsets outside + that domain are rejected; no new wire format or application migration is needed. + - **Serializable SPV Structures**: Structures and interfaces for full SPV verification. - **Secure Encryption and Signed Messages**: Enhanced mechanisms for encryption and digital signatures, replacing outdated methods. diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 089328862..9b804e5c7 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/sdk", - "version": "2.4.2", + "version": "2.5.0", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/sdk/src/transaction/MerklePath.ts b/packages/sdk/src/transaction/MerklePath.ts index ee3fd9a4e..1eb80ddf0 100644 --- a/packages/sdk/src/transaction/MerklePath.ts +++ b/packages/sdk/src/transaction/MerklePath.ts @@ -10,6 +10,23 @@ export interface MerklePathLeaf { duplicate?: boolean } +function assertOffset(offset: number): void { + if (!Number.isSafeInteger(offset) || offset < 0) { + throw new Error('Invalid offset') + } +} + +// JavaScript bitwise operators truncate offsets to signed 32 bits. Division by +// a power of two is exact over the supported safe-integer offset domain. +const offsetAtHeight = (offset: number, height: number): number => Math.floor(offset / 2 ** height) + +const siblingOf = (offset: number): number => offset % 2 === 0 ? offset + 1 : offset - 1 + +const sameNodeAtHeight = (index: number, maxOffset: number, height: number): boolean => + offsetAtHeight(index, height) === offsetAtHeight(maxOffset, height) + +const offsetTreeHeight = (offset: number): number => offset === 0 ? 0 : offset.toString(2).length + function hashPair(left: string | undefined, right: string | undefined): string { return toHex(hash256(toArray((left ?? '') + (right ?? ''), 'hex').reverse()).reverse()) } @@ -191,6 +208,7 @@ export default class MerklePath { } const offsetsAtThisHeight = new Set() leaves.forEach(leaf => { + assertOffset(leaf.offset) if (offsetsAtThisHeight.has(leaf.offset)) { throw new Error(`Duplicate offset: ${leaf.offset}, at height: ${height}`) } @@ -198,7 +216,7 @@ export default class MerklePath { if (height === 0) { if (leaf.duplicate !== true) { for (let h = 1; h < this.path.length; h++) { - legalOffsets[h].add((leaf.offset >> h) ^ 1) + legalOffsets[h].add(siblingOf(offsetAtHeight(leaf.offset, h))) } } } else if (legalOffsetsOnly && !legalOffsets[height].has(leaf.offset)) { @@ -317,14 +335,14 @@ export default class MerklePath { if (typeof txid !== 'string') throw new TypeError('Transaction ID is undefined') const index = this.indexOf(txid) if (this.path.length === 1 && this.path[0].length === 1) return txid - const treeHeight = Math.max(this.path.length, 32 - Math.clz32(maxOffset)) + const treeHeight = Math.max(this.path.length, offsetTreeHeight(maxOffset)) let workingHash = txid for (let height = 0; height < treeHeight; height++) { - const nodeKey = `${height}:${index >> height}` + const nodeKey = `${height}:${offsetAtHeight(index, height)}` const cachedRoot = cachedMerkleRoot(nodeKey, workingHash, treeHeight, nodeHashCache) if (cachedRoot != null) return cachedRoot nodeHashCache.set(nodeKey, workingHash) - const offset = (index >> height) ^ 1 + const offset = siblingOf(offsetAtHeight(index, height)) const leaf = this.cachedFindLeaf(height, offset, sourceIndex, hashCache, maxOffset) workingHash = nextCachedHash( workingHash, @@ -332,32 +350,13 @@ export default class MerklePath { offset, index, height, - this.path.length === 1 && index >> height === maxOffset >> height + this.path.length === 1 && sameNodeAtHeight(index, maxOffset, height) ) } nodeHashCache.set(`${treeHeight}:0`, workingHash) return workingHash } - private nextRootHash( - workingHash: string, - index: number, - height: number, - maxOffset: number - ): string { - const offset = (index >> height) ^ 1 - const leaf = this.findOrComputeLeaf(height, offset) - if (leaf == null) { - const isLastOddNode = this.path.length === 1 && index >> height === maxOffset >> height - if (isLastOddNode) return hashPair(workingHash, workingHash) - throw new Error(`Missing hash for index ${index} at height ${height}`) - } - if (leaf.duplicate === true) return hashPair(workingHash, workingHash) - return offset % 2 === 1 - ? hashPair(leaf.hash, workingHash) - : hashPair(workingHash, leaf.hash) - } - /** * Computes the Merkle root from the provided transaction ID. * @@ -387,10 +386,18 @@ export default class MerklePath { // (path.length === 1 or intermediate levels are empty/trimmed), we need to compute up // to the height implied by the highest offset present in path[0]. const maxOffset = this.path[0].reduce((max, l) => Math.max(max, l.offset), 0) - const treeHeight = Math.max(this.path.length, 32 - Math.clz32(maxOffset)) + const treeHeight = Math.max(this.path.length, offsetTreeHeight(maxOffset)) for (let height = 0; height < treeHeight; height++) { - workingHash = this.nextRootHash(workingHash, index, height, maxOffset) + const offset = siblingOf(offsetAtHeight(index, height)) + workingHash = nextCachedHash( + workingHash, + this.findOrComputeLeaf(height, offset), + offset, + index, + height, + this.path.length === 1 && sameNodeAtHeight(index, maxOffset, height) + ) } return workingHash } @@ -404,7 +411,7 @@ export default class MerklePath { * @param offset */ findOrComputeLeaf(height: number, offset: number): MerklePathLeaf | undefined { - const hash = (m: string): string => toHex(hash256(toArray(m, 'hex').reverse()).reverse()) + assertOffset(offset) let leaf: MerklePathLeaf | undefined = height < this.path.length ? this.path[height].find(l => l.offset === offset) : undefined @@ -414,7 +421,9 @@ export default class MerklePath { if (height === 0) return undefined const h = height - 1 - const l = offset << 1 + const l = offset * 2 + // No descendant of this node is representable by a supported leaf offset. + if (!Number.isSafeInteger(l)) return undefined const leaf0 = this.findOrComputeLeaf(h, l) if (leaf0?.hash == null || leaf0.hash === '') return undefined @@ -423,13 +432,13 @@ export default class MerklePath { if (leaf1?.hash == null) { // Explicit duplicate marker — duplicate leaf0 regardless of path depth. if (leaf1?.duplicate === true) { - return { offset, hash: hash(leaf0.hash + leaf0.hash) } + return { offset, hash: hashPair(leaf0.hash, leaf0.hash) } } // For single-level paths, leaf0 may be the last odd node at height h — duplicate it. if (this.path.length === 1) { const maxOffset0 = this.path[0].reduce((max, lf) => Math.max(max, lf.offset), 0) - if (l === maxOffset0 >> h) { - return { offset, hash: hash(leaf0.hash + leaf0.hash) } + if (l === offsetAtHeight(maxOffset0, h)) { + return { offset, hash: hashPair(leaf0.hash, leaf0.hash) } } } return undefined @@ -437,9 +446,9 @@ export default class MerklePath { let workinghash: string if (leaf1.duplicate === true) { - workinghash = hash(leaf0.hash + leaf0.hash) + workinghash = hashPair(leaf0.hash, leaf0.hash) } else { - workinghash = hash((leaf1.hash ?? '') + (leaf0.hash ?? '')) + workinghash = hashPair(leaf1.hash, leaf0.hash) } leaf = { offset, @@ -530,7 +539,7 @@ export default class MerklePath { const nextComputedOffsets = (cos: number[]): number[] => { const ncos: number[] = [] for (const o of cos) { - pushIfNew(o >> 1, ncos) + pushIfNew(offsetAtHeight(o, 1), ncos) } return ncos } @@ -545,7 +554,7 @@ export default class MerklePath { const n = this.path[0][l] if (n.txid === true) { // level 0 must enable computing level 1 for txid nodes - pushIfNew(n.offset >> 1, computedOffsets) + pushIfNew(offsetAtHeight(n.offset, 1), computedOffsets) } else { const isOdd = n.offset % 2 === 1 const peer = this.path[0][l + (isOdd ? -1 : 1)] @@ -577,7 +586,6 @@ export default class MerklePath { const key = `${height}:${offset}` if (hashCache.has(key)) return hashCache.get(key) - const doHash = (m: string): string => toHex(hash256(toArray(m, 'hex').reverse()).reverse()) let leaf: MerklePathLeaf | undefined = height < sourceIndex.length ? sourceIndex[height].get(offset) : undefined @@ -593,7 +601,8 @@ export default class MerklePath { } const h = height - 1 - const l = offset << 1 + const l = offset * 2 + if (!Number.isSafeInteger(l)) return undefined const leaf0 = this.cachedFindLeaf(h, l, sourceIndex, hashCache, maxOffset) if (leaf0?.hash == null || leaf0.hash === '') { hashCache.set(key, undefined) @@ -602,8 +611,8 @@ export default class MerklePath { const leaf1 = this.cachedFindLeaf(h, l + 1, sourceIndex, hashCache, maxOffset) if (leaf1?.hash == null) { - if (leaf1?.duplicate === true || (this.path.length === 1 && l === maxOffset >> h)) { - leaf = { offset, hash: doHash(leaf0.hash + leaf0.hash) } + if (leaf1?.duplicate === true || (this.path.length === 1 && l === offsetAtHeight(maxOffset, h))) { + leaf = { offset, hash: hashPair(leaf0.hash, leaf0.hash) } hashCache.set(key, leaf) return leaf } @@ -613,8 +622,8 @@ export default class MerklePath { const workinghash = leaf1.duplicate === true - ? doHash(leaf0.hash + leaf0.hash) - : doHash((leaf1.hash ?? '') + (leaf0.hash ?? '')) + ? hashPair(leaf0.hash, leaf0.hash) + : hashPair(leaf1.hash, leaf0.hash) leaf = { offset, hash: workinghash } hashCache.set(key, leaf) return leaf @@ -648,7 +657,7 @@ export default class MerklePath { const originalRoot = this.computeRoot() const maxOffset = this.path[0].reduce((max, l) => Math.max(max, l.offset), 0) - const treeHeight = Math.max(this.path.length, 32 - Math.clz32(maxOffset)) + const treeHeight = Math.max(this.path.length, offsetTreeHeight(maxOffset)) const sourceIndex = this.createSourceLeafIndex() const hashCache = new Map() @@ -720,7 +729,7 @@ export default class MerklePath { // Level 0: the txid leaf + its sibling. neededPerLevel[0].set(txOffset, { offset: txOffset, txid: true, hash: txid }) - const levelZeroSiblingOffset = txOffset ^ 1 + const levelZeroSiblingOffset = siblingOf(txOffset) if (!neededPerLevel[0].has(levelZeroSiblingOffset)) { const sibling = this.cachedFindLeaf(0, levelZeroSiblingOffset, sourceIndex, hashCache, maxOffset) if (sibling != null) neededPerLevel[0].set(levelZeroSiblingOffset, sibling) @@ -728,12 +737,12 @@ export default class MerklePath { // Higher levels need only the sibling at each height. for (let h = 1; h < treeHeight; h++) { - const siblingOffset = (txOffset >> h) ^ 1 + const siblingOffset = siblingOf(offsetAtHeight(txOffset, h)) if (neededPerLevel[h].has(siblingOffset)) continue const sibling = this.cachedFindLeaf(h, siblingOffset, sourceIndex, hashCache, maxOffset) if (sibling != null) { neededPerLevel[h].set(siblingOffset, sibling) - } else if (txOffset >> h === maxOffset >> h) { + } else if (sameNodeAtHeight(txOffset, maxOffset, h)) { neededPerLevel[h].set(siblingOffset, { offset: siblingOffset, duplicate: true }) } } diff --git a/packages/sdk/src/transaction/__tests/MerklePath.safeOffsets.test.ts b/packages/sdk/src/transaction/__tests/MerklePath.safeOffsets.test.ts new file mode 100644 index 000000000..e0224db77 --- /dev/null +++ b/packages/sdk/src/transaction/__tests/MerklePath.safeOffsets.test.ts @@ -0,0 +1,234 @@ +import { createHash } from 'node:crypto' +import MerklePath, { type MerklePathLeaf } from '../MerklePath' + +const TARGET_PREFIX = 'safe-offset-target' + +function sha256d(bytes: Uint8Array): Buffer { + const first = createHash('sha256').update(bytes).digest() + return createHash('sha256').update(first).digest() +} + +function displayToInternal(hash: string): Buffer { + return Buffer.from(hash, 'hex').reverse() +} + +function internalToDisplay(bytes: Uint8Array): string { + return Buffer.from(bytes).reverse().toString('hex') +} + +function hashPair(left: string, right: string): string { + return internalToDisplay( + sha256d(Buffer.concat([displayToInternal(left), displayToInternal(right)])) + ) +} + +function labelledHash(label: string): string { + return createHash('sha256').update(label).digest('hex') +} + +function bitLength(value: bigint): number { + let remaining = value + let length = 0 + while (remaining > 0n) { + remaining >>= 1n + length++ + } + return length +} + +function compact(value: bigint): number[] { + if (value < 253n) return [Number(value)] + if (value < 0x10000n) return [0xfd, Number(value & 0xffn), Number((value >> 8n) & 0xffn)] + if (value < 0x100000000n) { + return [ + 0xfe, + ...Array.from({ length: 4 }, (_, index) => Number((value >> BigInt(index * 8)) & 0xffn)) + ] + } + return [ + 0xff, + ...Array.from({ length: 8 }, (_, index) => Number((value >> BigInt(index * 8)) & 0xffn)) + ] +} + +interface SyntheticPath { + readonly target: string + readonly sibling: string + readonly index: number + readonly depth: number + readonly path: MerklePathLeaf[][] + readonly root: string +} + +/** + * Builds a bounded mathematical proof: only the target, its paired level-zero + * sibling, and one sibling per higher level are represented. It does not claim + * to model a mined block with 2^n transactions. + */ +function pairedPath(originalIndex: bigint): SyntheticPath { + const index = Number(originalIndex) + const depth = bitLength(originalIndex) + 1 + const target = labelledHash(`${TARGET_PREFIX}:${originalIndex}:target`) + const sibling = labelledHash(`${TARGET_PREFIX}:${originalIndex}:sibling:0`) + const path: MerklePathLeaf[][] = [ + [ + { offset: index, hash: target, txid: true }, + { offset: Number(originalIndex ^ 1n), hash: sibling, txid: true } + ].sort((left, right) => left.offset - right.offset) + ] + let root = originalIndex % 2n === 0n ? hashPair(target, sibling) : hashPair(sibling, target) + + for (let height = 1; height < depth; height++) { + const node = originalIndex >> BigInt(height) + const siblingHash = labelledHash(`${TARGET_PREFIX}:${originalIndex}:sibling:${height}`) + path.push([{ offset: Number(node ^ 1n), hash: siblingHash }]) + root = node % 2n === 0n ? hashPair(root, siblingHash) : hashPair(siblingHash, root) + } + + return { target, sibling, index, depth, path, root } +} + +function canonicalOddWidthPath(originalIndex: bigint): SyntheticPath { + const index = Number(originalIndex) + const target = labelledHash(`${TARGET_PREFIX}:${originalIndex}:odd-target`) + const path: MerklePathLeaf[][] = [] + let node = originalIndex + let width = originalIndex + 1n + let root = target + + // The target is the final node in each odd-width level. A canonical BUMP + // duplicates it at the next offset until the width reduces to two nodes. + while (width > 2n) { + if (width % 2n !== 1n || node !== width - 1n) + throw new Error('Invalid synthetic odd-width fixture') + const duplicate = { offset: Number(node ^ 1n), duplicate: true } + if (path.length === 0) { + path.push( + [{ offset: index, hash: target, txid: true }, duplicate].sort( + (left, right) => left.offset - right.offset + ) + ) + } else { + path.push([duplicate]) + } + root = hashPair(root, root) + node >>= 1n + width = (width + 1n) >> 1n + } + const leftRoot = labelledHash(`${TARGET_PREFIX}:${originalIndex}:odd-left-root`) + path.push([{ offset: Number(node ^ 1n), hash: leftRoot }]) + root = node % 2n === 0n ? hashPair(root, leftRoot) : hashPair(leftRoot, root) + + return { target, sibling: target, index, depth: path.length, path, root } +} + +describe('MerklePath safe offsets', () => { + it('keeps a low-offset control proof compatible', () => { + const fixture = pairedPath(5n) + const merklePath = new MerklePath(777, fixture.path) + + expect(merklePath.computeRoot(fixture.target)).toBe(fixture.root) + expect(MerklePath.fromHex(merklePath.toHex()).computeRoot(fixture.target)).toBe(fixture.root) + }) + + it.each([ + (1n << 31n) - 1n, + 1n << 31n, + (1n << 32n) - 1n, + 1n << 32n, + (1n << 53n) - 2n, + (1n << 53n) - 1n + ])('preserves the full synthetic proof at original offset %s', originalIndex => { + const fixture = pairedPath(originalIndex) + const merklePath = new MerklePath(777, fixture.path) + + expect(merklePath.path).toHaveLength(fixture.depth) + expect(merklePath.path[0].map(leaf => leaf.offset)).toEqual( + fixture.path[0].map(leaf => leaf.offset) + ) + expect(merklePath.computeRoot(fixture.target)).toBe(fixture.root) + expect(merklePath.computeRoot(fixture.sibling)).toBe(fixture.root) + + const roundTripped = MerklePath.fromHex(merklePath.toHex()) + expect(roundTripped.path[0].map(leaf => leaf.offset)).toEqual( + fixture.path[0].map(leaf => leaf.offset) + ) + expect(roundTripped.computeRoot(fixture.target)).toBe(fixture.root) + expect(roundTripped.computeRoot(fixture.sibling)).toBe(fixture.root) + }) + + it.each([31, 32, 52])('computes a canonical odd-width proof at 2^%i', power => { + const fixture = canonicalOddWidthPath(1n << BigInt(power)) + const merklePath = new MerklePath(777, fixture.path) + + expect(fixture.depth).toBe(power + 1) + expect(merklePath.computeRoot(fixture.target)).toBe(fixture.root) + }) + + it.each([31, 32, 52])('round-trips a canonical odd-width proof at 2^%i', power => { + const fixture = canonicalOddWidthPath(1n << BigInt(power)) + const roundTripped = MerklePath.fromHex(new MerklePath(777, fixture.path).toHex()) + + expect(roundTripped.path).toEqual(fixture.path) + expect(roundTripped.computeRoot(fixture.target)).toBe(fixture.root) + }) + + it('computes recursive leaves, extracts, combines, and trims a pruned high-offset proof', () => { + const fixture = pairedPath((1n << 53n) - 1n) + const full = new MerklePath(777, fixture.path) + const parentOffset = Math.floor(fixture.index / 2) + const expectedParent = hashPair(fixture.sibling, fixture.target) + + expect(full.findOrComputeLeaf(1, parentOffset)?.hash).toBe(expectedParent) + + const first = full.extract([fixture.target]) + const second = full.extract([fixture.sibling]) + expect(first.computeRoot(fixture.target)).toBe(fixture.root) + expect(second.computeRoot(fixture.sibling)).toBe(fixture.root) + + first.combine(second) + first.trim() + expect(first.computeRoot(fixture.target)).toBe(fixture.root) + expect(first.computeRoot(fixture.sibling)).toBe(fixture.root) + + const roundTripped = MerklePath.fromBinary(first.toBinaryUint8Array()) + expect(roundTripped.computeRoot(fixture.target)).toBe(fixture.root) + expect(roundTripped.computeRoot(fixture.sibling)).toBe(fixture.root) + }) + + it('computes safe high offsets when constructor root validation is deferred', () => { + const fixture = pairedPath(1n << 32n) + const merklePath = new MerklePath(777, fixture.path, true, false) + + expect(merklePath.computeRoot(fixture.target)).toBe(fixture.root) + }) + + it.each([Number.MAX_SAFE_INTEGER + 1, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( + 'rejects unsafe offset %p even when root validation is disabled', + offset => { + expect( + () => + new MerklePath( + 777, + [[{ offset, hash: labelledHash(`invalid:${offset}`), txid: true }]], + true, + false + ) + ).toThrow() + } + ) + + it('rejects an unsafe offset encoded directly as a BUMP varint', () => { + const unsafeOffset = 1n << 53n + const bytes = Uint8Array.from([ + ...compact(777n), + 1, + ...compact(1n), + ...compact(unsafeOffset), + 2, + ...displayToInternal(labelledHash('unsafe-wire')) + ]) + + expect(() => MerklePath.fromBinary(bytes, true, false)).toThrow() + }) +}) From d633740578de4dad648937c1c5d5e7cfbbc6edb0 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 13:40:55 -0500 Subject: [PATCH 004/168] chore(sdk): align candidate version registry --- governance/repository-health/baselines.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index aad0934bd..aebb1eaf6 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -322,7 +322,7 @@ "@bsv/overlay-discovery-services": "2.2.1", "@bsv/overlay-express": "2.6.1", "@bsv/overlay-topics": "1.7.1", - "@bsv/sdk": "2.4.2", + "@bsv/sdk": "2.5.0", "@bsv/verifast": "0.3.5", "@bsv/btms": "1.2.2", "@bsv/btms-permission-module": "1.1.4", From cbccb6cff94fe99ac0d957843807fbe7ad22ff1c Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 14:48:42 -0500 Subject: [PATCH 005/168] feat(sdk): add dynamic overlay lookup discovery and bounded host scheduling Subscribe to SLAP tracker updates during an active query so later advertised hosts can join a fair, concurrency-bounded queue. Read lookup bodies incrementally with honest resource-limit accounting, keep trackers as routing sources, and leave raw query$ unverified while exposing the C02 onEvidence intake seam. Preserves 2s/5s delays, reputation/backoff, query/query$/freeform APIs, and existing CORS/public lookup request headers. --- packages/sdk/CHANGELOG.md | 6 + packages/sdk/docs/reference/overlay-tools.md | 851 ++++++++++++++- .../__tests__/BRC100OverlayTransport.test.ts | 43 +- .../sdk/src/overlay-tools/LookupDiscovery.ts | 101 ++ .../sdk/src/overlay-tools/LookupHostQueue.ts | 74 ++ .../sdk/src/overlay-tools/LookupResolver.ts | 994 +++++++++--------- .../sdk/src/overlay-tools/LookupResources.ts | 83 ++ .../src/overlay-tools/LookupResponseReader.ts | 189 ++++ .../__tests/LookupResolver.additional.test.ts | 55 +- .../__tests/LookupResolver.dynamic.test.ts | 841 +++++++++++++++ .../__tests/LookupResolver.http.test.ts | 272 +++++ .../__tests/LookupResolver.test.ts | 198 ++-- .../__tests/LookupResponseReader.test.ts | 243 +++++ 13 files changed, 3275 insertions(+), 675 deletions(-) create mode 100644 packages/sdk/src/overlay-tools/LookupDiscovery.ts create mode 100644 packages/sdk/src/overlay-tools/LookupHostQueue.ts create mode 100644 packages/sdk/src/overlay-tools/LookupResources.ts create mode 100644 packages/sdk/src/overlay-tools/LookupResponseReader.ts create mode 100644 packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts create mode 100644 packages/sdk/src/overlay-tools/__tests/LookupResolver.http.test.ts create mode 100644 packages/sdk/src/overlay-tools/__tests/LookupResponseReader.test.ts diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index d6b61d775..4f913e481 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -245,6 +245,12 @@ All notable changes to this project will be documented in this file. The format - Add `LookupResolver.queryDetailed()` and per-outcome host-settlement counts so security-sensitive callers can distinguish authoritative empty answers from partial availability. +- Add bounded LookupResolver discovery: later SLAP tracker advertisements can + join an active query, hosts are scheduled fairly under concurrency and byte + limits, and HTTP bodies are read incrementally. Raw `query$` output remains + unverified; `onEvidence` is the C02 intake seam. Existing 2s host / 5s + tracker delays, reputation/backoff, freeform answers, and CORS/public + lookup request headers are unchanged. ### Changed diff --git a/packages/sdk/docs/reference/overlay-tools.md b/packages/sdk/docs/reference/overlay-tools.md index ec167e2bb..7e7608f7a 100644 --- a/packages/sdk/docs/reference/overlay-tools.md +++ b/packages/sdk/docs/reference/overlay-tools.md @@ -4,16 +4,17 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ## Interfaces -| | -| --- | -| [AdmittanceInstructions](#interface-admittanceinstructions) | -| [LookupQuestion](#interface-lookupquestion) | -| [LookupResolverConfig](#interface-lookupresolverconfig) | -| [OverlayBroadcastFacilitator](#interface-overlaybroadcastfacilitator) | -| [OverlayLookupFacilitator](#interface-overlaylookupfacilitator) | -| [RankedHost](#interface-rankedhost) | -| [SHIPBroadcasterConfig](#interface-shipbroadcasterconfig) | -| [TaggedBEEF](#interface-taggedbeef) | +| | | +| --- | --- | +| [AdmittanceInstructions](#interface-admittanceinstructions) | [LookupResolverConfig](#interface-lookupresolverconfig) | +| [LookupAnswerProgress](#interface-lookupanswerprogress) | [LookupResponseReaderOptions](#interface-lookupresponsereaderoptions) | +| [LookupDiscoveryUpdate](#interface-lookupdiscoveryupdate) | [OverlayBroadcastFacilitator](#interface-overlaybroadcastfacilitator) | +| [LookupFreeformAnswer](#interface-lookupfreeformanswer) | [OverlayLookupFacilitator](#interface-overlaylookupfacilitator) | +| [LookupLimits](#interface-lookuplimits) | [RankedHost](#interface-rankedhost) | +| [LookupQueryOptions](#interface-lookupqueryoptions) | [SHIPBroadcasterConfig](#interface-shipbroadcasterconfig) | +| [LookupQuestion](#interface-lookupquestion) | [TaggedBEEF](#interface-taggedbeef) | +| [LookupRequestOptions](#interface-lookuprequestoptions) | [UnreachableHostInfo](#interface-unreachablehostinfo) | +| [LookupResolution](#interface-lookupresolution) | | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -58,6 +59,352 @@ outputsToAdmit: number[] Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Interface: LookupAnswerProgress + +```ts +export interface LookupAnswerProgress { + discoveryComplete?: boolean; + terminalReason?: "settled" | "deadline" | "cancelled" | "resource-limit"; + discoveredHosts?: number; + skippedHosts?: number; + receivedBytes?: number; + retainedBytes?: number; + evidenceBytes?: number; + trackersTotal?: number; + trackersCompleted?: number; + trackersFailed?: number; + limitsHit?: string[]; + type: "output-list"; + outputs: Array<{ + beef: number[]; + outputIndex: number; + context?: number[]; + txid?: string; + }>; + txIds: string[]; + isFinal: boolean; + hostCount: number; + completedHosts: number; + successfulHosts: number; + emptyHosts: number; + failedHosts: number; + rejectedHosts: number; + freeformHosts: number; + correlationId?: string; +} +``` + +#### Property completedHosts + +Number of hosts that have settled (success / fail / timeout). + +```ts +completedHosts: number +``` + +#### Property correlationId + +Correlation id used for privacy-safe distributed diagnostics. + +```ts +correlationId?: string +``` + +#### Property discoveryComplete + +Transport coverage only, never cryptographic validity or global absence. + +```ts +discoveryComplete?: boolean +``` + +#### Property emptyHosts + +Successful hosts whose output list was empty. + +```ts +emptyHosts: number +``` + +#### Property evidenceBytes + +Receipt-copy octets handed to onEvidence, independently bounded. + +```ts +evidenceBytes?: number +``` + +#### Property failedHosts + +Hosts that failed due to availability, timeout, or malformed responses. + +```ts +failedHosts: number +``` + +#### Property freeformHosts + +Hosts that returned a valid but non-aggregatable freeform response. + +```ts +freeformHosts: number +``` + +#### Property hostCount + +Number of ranked hosts that were queried. + +```ts +hostCount: number +``` + +#### Property isFinal + +True only for the final emission, after every in-flight host has settled. + +```ts +isFinal: boolean +``` + +#### Property rejectedHosts + +Hosts that rejected this query semantically (for example, HTTP 400). + +```ts +rejectedHosts: number +``` + +#### Property retainedBytes + +Retained decoded BEEF/context octets; JavaScript arrays have additional heap overhead. + +```ts +retainedBytes?: number +``` + +#### Property successfulHosts + +Hosts that returned a structurally valid output-list response. + +```ts +successfulHosts: number +``` + +#### Property txIds + +Parallel array of resolved tx ids for each output (same index as `outputs`). + +```ts +txIds: string[] +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Interface: LookupDiscoveryUpdate + +```ts +export interface LookupDiscoveryUpdate { + sources: Map; + trackersTotal: number; + trackersCompleted: number; + trackersFailed: number; + skippedHosts: number; + receivedBytes: number; + limitsHit: Set; + done: boolean; +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Interface: LookupFreeformAnswer + +A valid non-aggregatable response returned by a lookup service. + +```ts +export interface LookupFreeformAnswer { + type: "freeform"; + result: unknown; +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Interface: LookupLimits + +Operational client limits, not BEEF validity or service authority rules. + +```ts +export interface LookupLimits { + maxHosts: number; + maxHostsPerTracker: number; + maxTrackers: number; + hostConcurrency: number; + trackerConcurrency: number; + maxResponseBytes: number; + maxTotalBytes: number; + maxOutputs: number; + maxEvidenceOutputs: number; + maxEvidenceBytes: number; +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Interface: LookupQueryOptions + +```ts +export interface LookupQueryOptions { + signal?: AbortSignal; + evidenceLimits?: { + maxOutputs?: number; + maxBytes?: number; + }; + deadlineMs?: number; + limits?: Partial; + onEvidence?: (event: LookupEvidenceEvent) => void | Promise; + graceMs?: number; + softTimeoutMs?: number; + onUnreachableHost?: (info: UnreachableHostInfo) => void | Promise; + unreachableHostNotificationCooldownMs?: number; + holdForUnknownHosts?: boolean; + waitForAllHosts?: boolean; + correlationId?: string; +} +``` + +See also: [LookupEvidenceEvent](./overlay-tools.md#type-lookupevidenceevent), [LookupLimits](./overlay-tools.md#interface-lookuplimits), [UnreachableHostInfo](./overlay-tools.md#interface-unreachablehostinfo) + +#### Property correlationId + +Correlates resolver and downstream wallet telemetry without logging the query payload. + +```ts +correlationId?: string +``` + +#### Property deadlineMs + +Whole attempt budget including discovery and queued hosts. Default 10000 ms. + +```ts +deadlineMs?: number +``` + +#### Property evidenceLimits + +Callback intake budget, independent of legacy aggregation. Defaults to 512 +outputs / 16 MiB of BEEF and context bytes. Values must be positive safe +integers. Coordinate these with a downstream verifier's admission limits. + +```ts +evidenceLimits?: { + maxOutputs?: number; + maxBytes?: number; +} +``` + +#### Property graceMs + +Override the grace window (ms) between the first valid response and the resolution of the query. +Late responders arriving within this window are merged into the result. Default 80 ms. +Raise for identity-style paths (e.g. ~300 ms) where divergence between hosts matters. + +```ts +graceMs?: number +``` + +#### Property holdForUnknownHosts + +Compatibility alias for `waitForAllHosts`. Prefer `waitForAllHosts` in new +code. `waitForAllHosts` takes precedence when both are supplied. + +```ts +holdForUnknownHosts?: boolean +``` + +#### Property limits + +Per-query operational resource limits. These do not define evidence validity. + +```ts +limits?: Partial +``` +See also: [LookupLimits](./overlay-tools.md#interface-lookuplimits) + +#### Property onEvidence + +Owned, UNTRUSTED receipts before legacy txid/outpoint deduplication. Enqueue +promptly; callback completion is not awaited and failures are isolated. +Intake stops at the configured evidenceLimits, reporting one limit event. +No callbacks occur after the query iterator closes. Raw `query$` snapshots +remain unverified transport aggregates, not cryptographic proof. + +```ts +onEvidence?: (event: LookupEvidenceEvent) => void | Promise +``` +See also: [LookupEvidenceEvent](./overlay-tools.md#type-lookupevidenceevent) + +#### Property onUnreachableHost + +Fired when a SLAP-advertised host fails (network error, timeout, malformed +response). The resolver itself does not email or escalate — downstream +consumers (e.g. overlay-express) wire this up to the BSVA notification API +to let the originating overlay operator know about a stale advertisement. + +```ts +onUnreachableHost?: (info: UnreachableHostInfo) => void | Promise +``` +See also: [UnreachableHostInfo](./overlay-tools.md#interface-unreachablehostinfo) + +#### Property signal + +Abort this query without cancelling discovery still owned by another query. + +```ts +signal?: AbortSignal +``` + +#### Property softTimeoutMs + +Soft timeout (ms). When set: + - `query()` resolves with whatever has arrived as soon as any host answers, or after this timeout. + - `query$()` emits a (possibly empty) snapshot after this timeout if no host has answered yet, + then continues yielding late-host enrichments until the iterator is broken or final emission. + +```ts +softTimeoutMs?: number +``` + +#### Property unreachableHostNotificationCooldownMs + +Minimum interval between unreachable notifications for the same host and +service. Defaults to 60 seconds to prevent notification storms. Set to 0 +to disable deduplication. + +```ts +unreachableHostNotificationCooldownMs?: number +``` + +#### Property waitForAllHosts + +Wait for every queried host to settle before the first emission. This is +the default for `query()` because generic output cardinality is not proof +of freshness or authority. It defaults to `false` for progressive +`query$()` consumers. `holdForUnknownHosts` remains as a compatibility +alias; `waitForAllHosts` takes precedence when both are supplied. + +```ts +waitForAllHosts?: boolean +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ### Interface: LookupQuestion @@ -89,6 +436,37 @@ service: string Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Interface: LookupRequestOptions + +Optional bounded transport settings; older custom facilitators may ignore these. + +```ts +export interface LookupRequestOptions { + maxResponseBytes?: number; + maxOutputs?: number; + consumeBytes?: (bytes: number) => void; +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Interface: LookupResolution + +A lookup answer together with the host settlement evidence behind it. + +```ts +export interface LookupResolution { + answer: LookupAnswer; + progress: LookupAnswerProgress; +} +``` + +See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupAnswerProgress](./overlay-tools.md#interface-lookupanswerprogress) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ### Interface: LookupResolverConfig @@ -96,7 +474,8 @@ Configuration options for the Lookup resolver. ```ts export interface LookupResolverConfig { - networkPreset?: "mainnet" | "testnet" | "teratestnet" | "local"; + limits?: Partial; + networkPreset?: LookupNetworkPreset; facilitator?: OverlayLookupFacilitator; slapTrackers?: string[]; hostOverrides?: Record; @@ -106,10 +485,11 @@ export interface LookupResolverConfig { get: (key: string) => string | null | undefined; set: (key: string, value: string) => void; }; + telemetry?: TelemetryConfig; } ``` -See also: [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) +See also: [LookupLimits](./overlay-tools.md#interface-lookuplimits), [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset), [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) #### Property additionalHosts @@ -144,16 +524,27 @@ Map of lookup service names to arrays of hosts to use in place of resolving via hostOverrides?: Record ``` +#### Property limits + +Defaults for the bounded discovery, scheduler and receipt intake. + +```ts +limits?: Partial +``` +See also: [LookupLimits](./overlay-tools.md#interface-lookuplimits) + #### Property networkPreset The network preset to use, unless other options override it. - mainnet: use mainnet SLAP trackers and HTTPS facilitator - testnet: use testnet SLAP trackers and HTTPS facilitator +- teratestnet: use TerraTestNet SLAP trackers and HTTPS facilitator - local: directly query from localhost:8080 and a facilitator that permits plain HTTP ```ts -networkPreset?: "mainnet" | "testnet" | "teratestnet" | "local" +networkPreset?: LookupNetworkPreset ``` +See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset) #### Property reputationStorage @@ -174,6 +565,53 @@ The list of SLAP trackers queried to resolve Overlay Services hosts for a given slapTrackers?: string[] ``` +#### Property telemetry + +Optional privacy-bounded telemetry sink. Query payloads are never emitted. + +```ts +telemetry?: TelemetryConfig +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Interface: LookupResponseReaderOptions + +Options controlling a bounded lookup response read. + +```ts +export interface LookupResponseReaderOptions { + signal?: AbortSignal; + maxResponseBytes: number; + consumeBytes?: (bytes: number) => void; +} +``` + +#### Property consumeBytes + +Charges accepted bytes to the caller's aggregate response budget. + +```ts +consumeBytes?: (bytes: number) => void +``` + +#### Property maxResponseBytes + +Maximum number of response bytes to retain. + +```ts +maxResponseBytes: number +``` + +#### Property signal + +Cancels a pending stream read when the lookup request is aborted. + +```ts +signal?: AbortSignal +``` + Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- @@ -198,20 +636,20 @@ Facilitates lookups to URLs that return answers. ```ts export interface OverlayLookupFacilitator { - lookup: (url: string, question: LookupQuestion, timeout?: number) => Promise; + lookup: (url: string, question: LookupQuestion, timeout?: number, signal?: AbortSignal, options?: LookupRequestOptions) => Promise; } ``` -See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) +See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupRequestOptions](./overlay-tools.md#interface-lookuprequestoptions) #### Property lookup Returns a lookup answer for a lookup question ```ts -lookup: (url: string, question: LookupQuestion, timeout?: number) => Promise +lookup: (url: string, question: LookupQuestion, timeout?: number, signal?: AbortSignal, options?: LookupRequestOptions) => Promise ``` -See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) +See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupRequestOptions](./overlay-tools.md#interface-lookuprequestoptions) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -233,16 +671,16 @@ Configuration options for the SHIP broadcaster. ```ts export interface SHIPBroadcasterConfig { - networkPreset?: "mainnet" | "testnet" | "teratestnet" | "local"; + networkPreset?: LookupNetworkPreset; facilitator?: OverlayBroadcastFacilitator; resolver?: LookupResolver; - requireAcknowledgmentFromAllHostsForTopics?: "all" | "any" | string[]; - requireAcknowledgmentFromAnyHostForTopics?: "all" | "any" | string[]; - requireAcknowledgmentFromSpecificHostsForTopics?: Record; + requireAcknowledgmentFromAllHostsForTopics?: TopicAcknowledgmentRequirement; + requireAcknowledgmentFromAnyHostForTopics?: TopicAcknowledgmentRequirement; + requireAcknowledgmentFromSpecificHostsForTopics?: Record; } ``` -See also: [LookupResolver](./overlay-tools.md#class-lookupresolver), [OverlayBroadcastFacilitator](./overlay-tools.md#interface-overlaybroadcastfacilitator) +See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset), [LookupResolver](./overlay-tools.md#class-lookupresolver), [OverlayBroadcastFacilitator](./overlay-tools.md#interface-overlaybroadcastfacilitator), [TopicAcknowledgmentRequirement](./overlay-tools.md#type-topicacknowledgmentrequirement) #### Property facilitator @@ -258,35 +696,40 @@ See also: [OverlayBroadcastFacilitator](./overlay-tools.md#interface-overlaybroa The network preset to use, unless other options override it. - mainnet: use mainnet resolver and HTTPS facilitator - testnet: use testnet resolver and HTTPS facilitator +- teratestnet: use TerraTestNet resolver and HTTPS facilitator - local: directly send to localhost:8080 and a facilitator that permits plain HTTP ```ts -networkPreset?: "mainnet" | "testnet" | "teratestnet" | "local" +networkPreset?: LookupNetworkPreset ``` +See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset) #### Property requireAcknowledgmentFromAllHostsForTopics Determines which topics (all, any, or a specific list) must be present within all STEAKs received from every host for the broadcast to be considered a success. By default, all hosts must acknowledge all topics. ```ts -requireAcknowledgmentFromAllHostsForTopics?: "all" | "any" | string[] +requireAcknowledgmentFromAllHostsForTopics?: TopicAcknowledgmentRequirement ``` +See also: [TopicAcknowledgmentRequirement](./overlay-tools.md#type-topicacknowledgmentrequirement) #### Property requireAcknowledgmentFromAnyHostForTopics Determines which topics (all, any, or a specific list) must be present within STEAK received from at least one host for the broadcast to be considered a success. ```ts -requireAcknowledgmentFromAnyHostForTopics?: "all" | "any" | string[] +requireAcknowledgmentFromAnyHostForTopics?: TopicAcknowledgmentRequirement ``` +See also: [TopicAcknowledgmentRequirement](./overlay-tools.md#type-topicacknowledgmentrequirement) #### Property requireAcknowledgmentFromSpecificHostsForTopics Determines a mapping whose keys are specific hosts and whose values are the topics (all, any, or a specific list) that must be present within the STEAK received by the given hosts, in order for the broadcast to be considered a success. ```ts -requireAcknowledgmentFromSpecificHostsForTopics?: Record +requireAcknowledgmentFromSpecificHostsForTopics?: Record ``` +See also: [TopicAcknowledgmentRequirement](./overlay-tools.md#type-topicacknowledgmentrequirement) #### Property resolver @@ -314,6 +757,54 @@ export interface TaggedBEEF { Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Interface: UnreachableHostInfo + +Info supplied to onUnreachableHost callbacks. + +```ts +export interface UnreachableHostInfo { + host: string; + service: string; + error: string; + advertisedBy?: string; +} +``` + +#### Property advertisedBy + +SLAP tracker URL that advertised this host, if known. + +```ts +advertisedBy?: string +``` + +#### Property error + +Error message from the facilitator. + +```ts +error: string +``` + +#### Property host + +Host URL that failed. + +```ts +host: string +``` + +#### Property service + +Lookup service that was being queried when the failure occurred. + +```ts +service: string +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ## Classes @@ -322,7 +813,11 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | [HTTPSOverlayBroadcastFacilitator](#class-httpsoverlaybroadcastfacilitator) | | [HTTPSOverlayLookupFacilitator](#class-httpsoverlaylookupfacilitator) | | [HostReputationTracker](#class-hostreputationtracker) | +| [LookupDiscovery](#class-lookupdiscovery) | +| [LookupHTTPError](#class-lookuphttperror) | +| [LookupHostQueue](#class-lookuphostqueue) | | [LookupResolver](#class-lookupresolver) | +| [LookupResourceLimitError](#class-lookupresourcelimiterror) | | [OverlayAdminTokenTemplate](#class-overlayadmintokentemplate) | | [TopicBroadcaster](#class-topicbroadcaster) | @@ -353,11 +848,11 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { fetchClient: typeof fetch; allowHTTP: boolean; constructor(httpClient = defaultFetch, allowHTTP: boolean = false) - async lookup(url: string, question: LookupQuestion, timeout: number = 5000): Promise + async lookup(url: string, question: LookupQuestion, timeout: number = 2000, signal?: AbortSignal, options?: LookupRequestOptions): Promise } ``` -See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) +See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupRequestOptions](./overlay-tools.md#interface-lookuprequestoptions), [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -372,11 +867,72 @@ export class HostReputationTracker { recordFailure(host: string, reason?: unknown): void rankHosts(hosts: string[], now: number = Date.now()): RankedHost[] snapshot(host: string): HostReputationEntry | undefined + flush(): void } ``` See also: [RankedHost](./overlay-tools.md#interface-rankedhost) +#### Method flush + +Flushes a pending debounced persistence write immediately. + +```ts +flush(): void +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Class: LookupDiscovery + +One bounded refresh shared only by subscribers of the same resolver/configuration. + +```ts +export class LookupDiscovery { + readonly controller = new AbortController(); + readonly state: LookupDiscoveryUpdate; + constructor(private readonly trackers: string[], private readonly limits: LookupLimits, private readonly lookup: (tracker: string, signal: AbortSignal, consume: (bytes: number) => void) => Promise, private readonly finish: (state: LookupDiscoveryUpdate, abandoned: boolean) => void) + subscribe(listener: (state: LookupDiscoveryUpdate) => void): () => void +} +``` + +See also: [LookupDiscoveryUpdate](./overlay-tools.md#interface-lookupdiscoveryupdate), [LookupLimits](./overlay-tools.md#interface-lookuplimits) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Class: LookupHTTPError + +An HTTP failure with enough classification for reputation handling. + +```ts +export class LookupHTTPError extends Error { + readonly status: number; + readonly kind: LookupHTTPErrorKind; + constructor(status: number, kind: LookupHTTPErrorKind, statusText?: string) +} +``` + +See also: [LookupHTTPErrorKind](./overlay-tools.md#type-lookuphttperrorkind) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Class: LookupHostQueue + +A bounded FIFO within each source, round-robin between sources. + +```ts +export class LookupHostQueue { + readonly done = new Promise(resolve => { this.resolveDone = resolve; }); + constructor(private readonly maxHosts: number, private readonly concurrency: number, private readonly run: (host: string) => Promise, private readonly skipped: (count: number, limited: boolean) => void) + add(source: string, hosts: string[]): void + finishSources(): void + cancel(): void +} +``` + Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- @@ -387,20 +943,59 @@ Represents a Lookup Resolver. ```ts export default class LookupResolver { constructor(config: LookupResolverConfig = {}) - async query(question: LookupQuestion, timeout?: number): Promise + async query(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise + async queryDetailed(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise + query$(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): AsyncIterable } ``` -See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupResolverConfig](./overlay-tools.md#interface-lookupresolverconfig) +See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupAnswerProgress](./overlay-tools.md#interface-lookupanswerprogress), [LookupQueryOptions](./overlay-tools.md#interface-lookupqueryoptions), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupResolution](./overlay-tools.md#interface-lookupresolution), [LookupResolverConfig](./overlay-tools.md#interface-lookupresolverconfig) #### Method query Given a LookupQuestion, returns a LookupAnswer. Aggregates across multiple services and supports resiliency. +Optional `options.graceMs` overrides the per-call grace window (default 80 ms). +Optional `options.softTimeoutMs` resolves the query early with whatever has arrived once any host has +answered (or with an empty result if no host has answered by `softTimeoutMs`). + +```ts +async query(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise +``` +See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQueryOptions](./overlay-tools.md#interface-lookupqueryoptions), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) + +#### Method query$ + +Cumulative unverified results. Discovery remains subscribed while trackers +settle; each new host enters the bounded queue immediately. Caller abort, +deadline and iterator close release this query's ownership. + ```ts -async query(question: LookupQuestion, timeout?: number): Promise +query$(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): AsyncIterable +``` +See also: [LookupAnswerProgress](./overlay-tools.md#interface-lookupanswerprogress), [LookupQueryOptions](./overlay-tools.md#interface-lookupqueryoptions), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) + +#### Method queryDetailed + +Performs a lookup and returns both its answer and the host settlement +evidence required by security-sensitive consumers to distinguish an +authoritative empty result from an availability failure. + +```ts +async queryDetailed(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise +``` +See also: [LookupQueryOptions](./overlay-tools.md#interface-lookupqueryoptions), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupResolution](./overlay-tools.md#interface-lookupresolution) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Class: LookupResourceLimitError + +```ts +export class LookupResourceLimitError extends Error { + constructor(readonly limit: string) +} ``` -See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -564,6 +1159,66 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( --- ## Functions +| | +| --- | +| [lookupAbortError](#function-lookupaborterror) | +| [lookupLimits](#function-lookuplimits) | +| [normalizeLookupHost](#function-normalizelookuphost) | +| [readLookupResponseBytes](#function-readlookupresponsebytes) | +| [withDoubleSpendRetry](#function-withdoublespendretry) | +| [withLookupAbort](#function-withlookupabort) | + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Function: lookupAbortError + +```ts +export function lookupAbortError(): Error +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Function: lookupLimits + +```ts +export function lookupLimits(...overrides: Array | undefined>): LookupLimits +``` + +See also: [LookupLimits](./overlay-tools.md#interface-lookuplimits) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Function: normalizeLookupHost + +Preserve distinct paths and ports; remove only a final slash and URL fragments. + +```ts +export function normalizeLookupHost(host: string, allowParameters: boolean = false): string | null +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Function: readLookupResponseBytes + +Reads a lookup response incrementally while enforcing a per-response bound. + +This deliberately does not use Response.text(), json(), or arrayBuffer(), +because those APIs buffer the complete body before a limit can be enforced. + +```ts +export async function readLookupResponseBytes(response: Response, options: LookupResponseReaderOptions): Promise +``` + +See also: [LookupResponseReaderOptions](./overlay-tools.md#interface-lookupresponsereaderoptions) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- ### Function: withDoubleSpendRetry Executes an operation with automatic retry logic for double-spend errors. @@ -595,13 +1250,30 @@ If max retries exceeded or non-double-spend error occurs Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Function: withLookupAbort + +A non-cooperative transport cannot retain a cancelled waiter. + +```ts +export async function withLookupAbort(work: Promise, signal?: AbortSignal): Promise +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ## Types | | | --- | | [LookupAnswer](#type-lookupanswer) | +| [LookupEvidenceEvent](#type-lookupevidenceevent) | +| [LookupFacilitatorAnswer](#type-lookupfacilitatoranswer) | +| [LookupHTTPErrorKind](#type-lookuphttperrorkind) | +| [LookupNetworkPreset](#type-lookupnetworkpreset) | +| [RequireMode](#type-requiremode) | | [STEAK](#type-steak) | +| [TopicAcknowledgmentRequirement](#type-topicacknowledgmentrequirement) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -609,8 +1281,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Type: LookupAnswer -How the Overlay Services Engine responds to a Lookup Question. -It may comprise either an output list or a freeform response from the Lookup Service. +An aggregatable output-list answer returned by the resolver. ```ts export type LookupAnswer = { @@ -619,12 +1290,76 @@ export type LookupAnswer = { beef: number[]; outputIndex: number; context?: number[]; + txid?: string; }>; } ``` Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Type: LookupEvidenceEvent + +Additive evidence intake, independent of the legacy aggregated answer. + +```ts +export type LookupEvidenceEvent = { + type: "output"; + host: string; + output: LookupAnswer["outputs"][number]; +} | { + type: "limit"; +} +``` + +See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Type: LookupFacilitatorAnswer + +Responses a facilitator may return before the resolver aggregates them. + +```ts +export type LookupFacilitatorAnswer = LookupAnswer | LookupFreeformAnswer +``` + +See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupFreeformAnswer](./overlay-tools.md#interface-lookupfreeformanswer) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Type: LookupHTTPErrorKind + +```ts +export type LookupHTTPErrorKind = "semantic" | "availability" +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Type: LookupNetworkPreset + +Public overlay network presets understood by lookup and SHIP routing. + +```ts +export type LookupNetworkPreset = "mainnet" | "testnet" | "teratestnet" | "local" +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- +### Type: RequireMode + +The require mode for topic acknowledgment: all topics must be present, or any one suffices. + +```ts +export type RequireMode = "all" | "any" +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ### Type: STEAK @@ -638,6 +1373,19 @@ See also: [AdmittanceInstructions](./overlay-tools.md#interface-admittanceinstru Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Type: TopicAcknowledgmentRequirement + +Specifies which topics must be acknowledged: all, any, or a specific list. + +```ts +export type TopicAcknowledgmentRequirement = RequireMode | string[] +``` + +See also: [RequireMode](./overlay-tools.md#type-requiremode) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ## Enums @@ -645,14 +1393,38 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | | | --- | +| [DEFAULT_LOOKUP_LIMITS](#variable-default_lookup_limits) | | [DEFAULT_SLAP_TRACKERS](#variable-default_slap_trackers) | | [DEFAULT_TESTNET_SLAP_TRACKERS](#variable-default_testnet_slap_trackers) | +| [DEFAULT_TTN_SLAP_TRACKERS](#variable-default_ttn_slap_trackers) | | [getOverlayHostReputationTracker](#variable-getoverlayhostreputationtracker) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- +### Variable: DEFAULT_LOOKUP_LIMITS + +```ts +DEFAULT_LOOKUP_LIMITS: Readonly = Object.freeze({ + maxHosts: 256, + maxHostsPerTracker: 64, + maxTrackers: 16, + hostConcurrency: 8, + trackerConcurrency: 4, + maxResponseBytes: 32 * 1024 * 1024, + maxTotalBytes: 64 * 1024 * 1024, + maxOutputs: 4096, + maxEvidenceOutputs: 512, + maxEvidenceBytes: 16 * 1024 * 1024 +}) +``` + +See also: [LookupLimits](./overlay-tools.md#interface-lookuplimits) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- ### Variable: DEFAULT_SLAP_TRACKERS ```ts @@ -677,6 +1449,17 @@ DEFAULT_TESTNET_SLAP_TRACKERS: string[] = [ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) +--- +### Variable: DEFAULT_TTN_SLAP_TRACKERS + +```ts +DEFAULT_TTN_SLAP_TRACKERS: string[] = [ + "https://staging-overlay.babbage.systems" +] +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + --- ### Variable: getOverlayHostReputationTracker diff --git a/packages/sdk/src/__tests__/BRC100OverlayTransport.test.ts b/packages/sdk/src/__tests__/BRC100OverlayTransport.test.ts index f9f0e1881..9cd8c4511 100644 --- a/packages/sdk/src/__tests__/BRC100OverlayTransport.test.ts +++ b/packages/sdk/src/__tests__/BRC100OverlayTransport.test.ts @@ -2,11 +2,13 @@ import { HTTPSOverlayLookupFacilitator } from '../overlay-tools/LookupResolver' describe('overlay lookup BRC-100 byte compatibility', () => { it('keeps typed query bytes portable across the JSON request boundary', async () => { - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/json' }, - json: async () => ({ type: 'output-list', outputs: [] }) - }) + const mockFetch = jest + .fn() + .mockResolvedValue( + new Response(JSON.stringify({ type: 'output-list', outputs: [] }), { + headers: { 'content-type': 'application/json' } + }) + ) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) await facilitator.lookup('http://host', { @@ -21,14 +23,17 @@ describe('overlay lookup BRC-100 byte compatibility', () => { }) it('recovers historical numeric-key BEEF from a JSON response', async () => { - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/json' }, - json: async () => ({ - type: 'output-list', - outputs: [{ beef: { 0: 1, 1: 2, 2: 255 }, outputIndex: 0 }] - }) - }) + const mockFetch = jest + .fn() + .mockResolvedValue( + new Response( + JSON.stringify({ + type: 'output-list', + outputs: [{ beef: { 0: 1, 1: 2, 2: 255 }, outputIndex: 0 }] + }), + { headers: { 'content-type': 'application/json' } } + ) + ) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) await expect( @@ -41,11 +46,13 @@ describe('overlay lookup BRC-100 byte compatibility', () => { it('preserves byte-like objects inside freeform lookup results', async () => { const result = { data: { 0: 1, 1: 2 }, tx: {}, payload: { 0: 3 } } - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/json' }, - json: async () => ({ type: 'freeform', result }) - }) + const mockFetch = jest + .fn() + .mockResolvedValue( + new Response(JSON.stringify({ type: 'freeform', result }), { + headers: { 'content-type': 'application/json' } + }) + ) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) await expect( diff --git a/packages/sdk/src/overlay-tools/LookupDiscovery.ts b/packages/sdk/src/overlay-tools/LookupDiscovery.ts new file mode 100644 index 000000000..12dbe6641 --- /dev/null +++ b/packages/sdk/src/overlay-tools/LookupDiscovery.ts @@ -0,0 +1,101 @@ +import { LookupLimits, LookupResourceLimitError, normalizeLookupHost } from './LookupResources.js' + +export interface LookupDiscoveryUpdate { + sources: Map + trackersTotal: number + trackersCompleted: number + trackersFailed: number + skippedHosts: number + receivedBytes: number + limitsHit: Set + done: boolean +} + +/** One bounded refresh shared only by subscribers of the same resolver/configuration. */ +export class LookupDiscovery { + readonly controller = new AbortController() + readonly state: LookupDiscoveryUpdate + private readonly listeners = new Set<(state: LookupDiscoveryUpdate) => void>() + private started = false + private abandoned = false + + constructor( + private readonly trackers: string[], + private readonly limits: LookupLimits, + private readonly lookup: (tracker: string, signal: AbortSignal, consume: (bytes: number) => void) => Promise, + private readonly finish: (state: LookupDiscoveryUpdate, abandoned: boolean) => void + ) { + this.state = { + sources: new Map(), trackersTotal: trackers.length, trackersCompleted: 0, + trackersFailed: 0, skippedHosts: 0, receivedBytes: 0, limitsHit: new Set(), done: false + } + } + + subscribe(listener: (state: LookupDiscoveryUpdate) => void): () => void { + this.listeners.add(listener) + listener(this.state) + if (!this.started) { this.started = true; void this.run() } + return () => { + this.listeners.delete(listener) + if (this.listeners.size === 0 && !this.state.done) { + this.abandoned = true + this.controller.abort() + this.finish(this.state, true) + } + } + } + + private emit(): void { + if (this.abandoned) return + for (const listener of this.listeners) listener(this.state) + } + + private consume = (bytes: number): void => { + if (this.abandoned) throw new LookupResourceLimitError('abandoned') + if (bytes > this.limits.maxTotalBytes - this.state.receivedBytes) { + this.state.limitsHit.add('maxTotalBytes') + throw new LookupResourceLimitError('maxTotalBytes') + } + this.state.receivedBytes += bytes + this.emit() + } + + private async run(): Promise { + let cursor = 0 + // Each tracker keeps a reserved share, so an early advertisement flood + // cannot consume the complete candidate budget before later sources reply. + const share = Math.min(this.limits.maxHostsPerTracker, + Math.max(1, Math.floor(this.limits.maxHosts / Math.max(1, this.trackers.length)))) + const worker = async (): Promise => { + while (!this.controller.signal.aborted && cursor < this.trackers.length) { + const tracker = this.trackers[cursor++] + try { + const candidates = await this.lookup(tracker, this.controller.signal, this.consume) + if (this.abandoned) return + const hosts = new Set() + for (const candidate of candidates) { + const host = normalizeLookupHost(candidate) + if (host === null) { this.state.skippedHosts++; continue } + if (hosts.has(host)) continue + if (hosts.size >= share) { + this.state.skippedHosts++ + this.state.limitsHit.add('maxHostsPerTracker') + } else hosts.add(host) + } + this.state.sources.set(tracker, Array.from(hosts)) + } catch (error) { + if (error instanceof LookupResourceLimitError) this.state.limitsHit.add(error.limit) + else if (!this.controller.signal.aborted) this.state.trackersFailed++ + } finally { + this.state.trackersCompleted++ + this.emit() + } + } + } + await Promise.all(Array.from({ length: Math.min(this.limits.trackerConcurrency, this.trackers.length) }, worker)) + this.state.done = true + this.finish(this.state, this.abandoned) + this.emit() + this.listeners.clear() + } +} diff --git a/packages/sdk/src/overlay-tools/LookupHostQueue.ts b/packages/sdk/src/overlay-tools/LookupHostQueue.ts new file mode 100644 index 000000000..6e5d86cf4 --- /dev/null +++ b/packages/sdk/src/overlay-tools/LookupHostQueue.ts @@ -0,0 +1,74 @@ +/** A bounded FIFO within each source, round-robin between sources. */ +export class LookupHostQueue { + private readonly queues = new Map() + private readonly seen = new Set() + private cursor = 0 + private active = 0 + private closed = false + private sourceClosed = false + private resolveDone: () => void = () => {} + readonly done = new Promise(resolve => { this.resolveDone = resolve }) + + constructor( + private readonly maxHosts: number, + private readonly concurrency: number, + private readonly run: (host: string) => Promise, + private readonly skipped: (count: number, limited: boolean) => void + ) {} + + add(source: string, hosts: string[]): void { + if (this.closed || this.sourceClosed) return + const queue = this.queues.get(source) ?? [] + this.queues.set(source, queue) + for (const host of hosts) { + if (this.seen.has(host)) continue + if (this.seen.size >= this.maxHosts) { this.skipped(1, true); continue } + this.seen.add(host) + queue.push(host) + } + this.pump() + } + + finishSources(): void { + this.sourceClosed = true + this.pump() + } + + cancel(): void { + if (this.closed) return + this.closed = true + for (const queue of this.queues.values()) { + this.skipped(queue.length, false) + queue.length = 0 + } + this.settle() + } + + private next(): string | undefined { + const sources = Array.from(this.queues.values()) + for (let i = 0; i < sources.length; i++) { + const index = this.cursor++ % sources.length + const host = sources[index].shift() + if (host !== undefined) return host + } + return undefined + } + + private settle(): void { + if ((this.closed || this.sourceClosed) && this.active === 0 && + Array.from(this.queues.values()).every(queue => queue.length === 0)) this.resolveDone() + } + + private pump(): void { + while (!this.closed && this.active < this.concurrency) { + const host = this.next() + if (host === undefined) break + this.active++ + void this.run(host).catch(() => {}).finally(() => { + this.active-- + this.pump() + }) + } + this.settle() + } +} diff --git a/packages/sdk/src/overlay-tools/LookupResolver.ts b/packages/sdk/src/overlay-tools/LookupResolver.ts index 66f44f1bc..c0e674110 100644 --- a/packages/sdk/src/overlay-tools/LookupResolver.ts +++ b/packages/sdk/src/overlay-tools/LookupResolver.ts @@ -1,7 +1,14 @@ +import { LookupDiscovery, LookupDiscoveryUpdate } from './LookupDiscovery.js' +import { LookupHostQueue } from './LookupHostQueue.js' +import { readLookupResponseBytes } from './LookupResponseReader.js' +import { DEFAULT_LOOKUP_LIMITS, LookupLimits, LookupResourceLimitError, lookupLimits, normalizeLookupHost, lookupAbortError, withLookupAbort } from './LookupResources.js' +export type { LookupLimits } from './LookupResources.js' +export { DEFAULT_LOOKUP_LIMITS, LookupResourceLimitError } from './LookupResources.js' import { Transaction } from '../transaction/index.js' import { Beef } from '../transaction/Beef.js' import OverlayAdminTokenTemplate from './OverlayAdminTokenTemplate.js' import * as Utils from '../primitives/utils.js' +import { sha256 } from '../primitives/Hash.js' import { getOverlayHostReputationTracker, HostReputationTracker } from './HostReputationTracker.js' import { Telemetry, TelemetryConfig } from '../telemetry/Telemetry.js' import { normalizeBRC100ByteFields, stringifyBRC100 } from '../wallet/BRC100ByteEncoding.js' @@ -53,6 +60,26 @@ export type LookupFacilitatorAnswer = LookupAnswer | LookupFreeformAnswer * All optional; defaults preserve prior behavior. */ export interface LookupQueryOptions { + /** Abort this query without cancelling discovery still owned by another query. */ + signal?: AbortSignal + /** + * Callback intake budget, independent of legacy aggregation. Defaults to 512 + * outputs / 16 MiB of BEEF and context bytes. Values must be positive safe + * integers. Coordinate these with a downstream verifier's admission limits. + */ + evidenceLimits?: { maxOutputs?: number; maxBytes?: number } + /** Whole attempt budget including discovery and queued hosts. Default 10000 ms. */ + deadlineMs?: number + /** Per-query operational resource limits. These do not define evidence validity. */ + limits?: Partial + /** + * Owned, UNTRUSTED receipts before legacy txid/outpoint deduplication. Enqueue + * promptly; callback completion is not awaited and failures are isolated. + * Intake stops at the configured evidenceLimits, reporting one limit event. + * No callbacks occur after the query iterator closes. Raw `query$` snapshots + * remain unverified transport aggregates, not cryptographic proof. + */ + onEvidence?: (event: LookupEvidenceEvent) => void | Promise /** * Override the grace window (ms) between the first valid response and the resolution of the query. * Late responders arriving within this window are merged into the result. Default 80 ms. @@ -96,6 +123,11 @@ export interface LookupQueryOptions { correlationId?: string } +/** Additive evidence intake, independent of the legacy aggregated answer. */ +export type LookupEvidenceEvent = + | { type: 'output'; host: string; output: LookupAnswer['outputs'][number] } + | { type: 'limit' } + /** Info supplied to onUnreachableHost callbacks. */ export interface UnreachableHostInfo { /** Host URL that failed. */ @@ -114,6 +146,20 @@ export interface UnreachableHostInfo { * and refine in place as more hosts answer. */ export interface LookupAnswerProgress { + /** Transport coverage only, never cryptographic validity or global absence. */ + discoveryComplete?: boolean + terminalReason?: 'settled' | 'deadline' | 'cancelled' | 'resource-limit' + discoveredHosts?: number + skippedHosts?: number + receivedBytes?: number + /** Retained decoded BEEF/context octets; JavaScript arrays have additional heap overhead. */ + retainedBytes?: number + /** Receipt-copy octets handed to onEvidence, independently bounded. */ + evidenceBytes?: number + trackersTotal?: number + trackersCompleted?: number + trackersFailed?: number + limitsHit?: string[] type: 'output-list' outputs: Array<{ beef: number[]; outputIndex: number; context?: number[]; txid?: string }> /** Parallel array of resolved tx ids for each output (same index as `outputs`). */ @@ -308,6 +354,8 @@ interface CacheOptions { /** Configuration options for the Lookup resolver. */ export interface LookupResolverConfig { + /** Defaults for the bounded discovery, scheduler and receipt intake. */ + limits?: Partial /** * The network preset to use, unless other options override it. * - mainnet: use mainnet SLAP trackers and HTTPS facilitator @@ -346,10 +394,19 @@ export interface OverlayLookupFacilitator { lookup: ( url: string, question: LookupQuestion, - timeout?: number + timeout?: number, + signal?: AbortSignal, + options?: LookupRequestOptions ) => Promise } +/** Optional bounded transport settings; older custom facilitators may ignore these. */ +export interface LookupRequestOptions { + maxResponseBytes?: number + maxOutputs?: number + consumeBytes?: (bytes: number) => void +} + export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { fetchClient: typeof fetch allowHTTP: boolean @@ -368,13 +425,18 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { async lookup( url: string, question: LookupQuestion, - timeout: number = 2000 + timeout: number = 2000, + signal?: AbortSignal, + options?: LookupRequestOptions ): Promise { if (!url.startsWith('https:') && !this.allowHTTP) { throw new Error('HTTPS facilitator can only use URLs that start with "https:"') } const controller = typeof AbortController === 'undefined' ? undefined : new AbortController() + if (signal?.aborted === true) throw lookupAbortError() + const abort = (): void => controller?.abort() + signal?.addEventListener('abort', abort, { once: true }) const deadline = createDeadline(timeout, controller) // Hard wall-clock deadline: in some environments (e.g. browser/Electron CORS @@ -382,25 +444,29 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { // AbortController signal alone is insufficient to make the returned promise // resolve or reject. Race the fetch against a setTimeout-backed reject so // the consumer-facing promise always settles within `timeout` ms. - const fetchPromise = this.performLookupRequest(url, question, controller?.signal) + const fetchPromise = this.performLookupRequest(url, question, controller?.signal, options) // Swallow background rejection if the deadline wins first. fetchPromise.catch(() => { /* noop */ }) try { - return await Promise.race([fetchPromise, deadline.promise]) + return await withLookupAbort(Promise.race([fetchPromise, deadline.promise]), signal) } catch (e) { + if (signal?.aborted) throw lookupAbortError() throw normalizeLookupError(e, deadline.didTimeOut()) } finally { deadline.cancel() + signal?.removeEventListener('abort', abort) + controller?.abort() } } private async performLookupRequest( url: string, question: LookupQuestion, - signal: AbortSignal | undefined + signal: AbortSignal | undefined, + options?: LookupRequestOptions ): Promise { const fco: RequestInit = { method: 'POST', @@ -412,7 +478,9 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { signal } const response: Response = await this.fetchClient(`${url}/lookup`, fco) - if (!response.ok) { + if (signal?.aborted === true || !response.ok) { + try { void response.body?.cancel().catch(() => {}) } catch { /* best-effort body cleanup */ } + if (signal?.aborted === true) throw lookupAbortError() // 408/429 are availability/backpressure signals. Other 4xx responses // reject this request but do not prove that the host is unavailable, so // they remain distinguishable and neutral for availability reputation. @@ -426,10 +494,15 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { : 'semantic' throw new LookupHTTPError(response.status, kind, response.statusText) } + const payload = await readLookupResponseBytes(response, { + signal, + maxResponseBytes: options?.maxResponseBytes ?? DEFAULT_LOOKUP_LIMITS.maxResponseBytes, + consumeBytes: options?.consumeBytes + }) if (isOctetStream(response.headers.get('content-type'))) { - return await this.parseOctetStreamLookup(response) + return await this.parseOctetStreamLookup(payload, signal, options) } - const answer = await response.json() + const answer = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(payload)) if ( answer != null && typeof answer === 'object' && @@ -437,6 +510,7 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { answer.type === 'output-list' && Array.isArray(answer.outputs) ) { + if (answer.outputs.length > (options?.maxOutputs ?? DEFAULT_LOOKUP_LIMITS.maxOutputs)) throw new LookupResourceLimitError('maxOutputs') for (const output of answer.outputs) { normalizeBRC100ByteFields(output, ['beef', 'context']) } @@ -445,10 +519,10 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { } /** Parse the aggregated octet-stream lookup response into an output-list LookupAnswer. */ - private async parseOctetStreamLookup(response: Response): Promise { - const payload = await response.arrayBuffer() - const r = new Utils.Reader([...new Uint8Array(payload)]) + private async parseOctetStreamLookup(payload: Uint8Array, signal?: AbortSignal, options?: LookupRequestOptions): Promise { + const r = new Utils.Reader(Array.from(payload)) const nOutpoints = r.readVarIntNum() + if (!Number.isSafeInteger(nOutpoints) || nOutpoints < 0 || nOutpoints > (options?.maxOutputs ?? DEFAULT_LOOKUP_LIMITS.maxOutputs)) throw new LookupResourceLimitError('maxOutputs') const outpoints: Array<{ txid: string; outputIndex: number; context?: number[] }> = [] for (let i = 0; i < nOutpoints; i++) { const txid = Utils.toHex(r.read(32)) @@ -459,14 +533,16 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { } const beef = r.read() const beefObj = Beef.fromBinary(beef) - const outputs = await this.extractAtomicOutputs(outpoints, beefObj) + const outputs = await this.extractAtomicOutputs(outpoints, beefObj, signal, options) return { type: 'output-list', outputs } } /** Memoize per-txid atomic BEEF extraction, yielding to the event loop between outputs. */ private async extractAtomicOutputs( outpoints: Array<{ txid: string; outputIndex: number; context?: number[] }>, - beefObj: Beef + beefObj: Beef, + signal?: AbortSignal, + options?: LookupRequestOptions ): Promise> { const beefByTxid = new Map() const outputs: Array<{ @@ -475,13 +551,17 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { beef: number[] txid: string }> = Array.from({ length: outpoints.length }) + let extractedBytes = 0 for (let idx = 0; idx < outpoints.length; idx++) { + if (signal?.aborted === true) throw lookupAbortError() const x = outpoints[idx] let beefBytes = beefByTxid.get(x.txid) if (beefBytes === undefined) { beefBytes = beefObj.toBinaryAtomic(x.txid) beefByTxid.set(x.txid, beefBytes) } + extractedBytes += beefBytes.length + (x.context?.length ?? 0) + if (extractedBytes > (options?.maxResponseBytes ?? DEFAULT_LOOKUP_LIMITS.maxResponseBytes)) throw new LookupResourceLimitError('maxResponseBytes') outputs[idx] = { outputIndex: x.outputIndex, context: x.context, @@ -497,28 +577,21 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { } } -type LookupQueryEvent = - | { kind: 'answer'; answer: LookupAnswer } - | { kind: 'done' } - | { kind: 'grace' } - | { kind: 'soft' } - interface LookupQuerySessionOptions { - hostCount: number graceMs: number softTimeoutMs?: number waitForAllHosts: boolean correlationId?: string - resolveTxId: ( - output: LookupAnswer['outputs'][number], - now: number - ) => string | null + limits: LookupLimits + onEvidence?: LookupQueryOptions['onEvidence'] + resolveTxId: (output: LookupAnswer['outputs'][number], now: number) => string | null } +/** A single cumulative snapshot plus a wake flag, regardless of listener speed. */ class LookupQuerySession { readonly startedAt = Date.now() - readonly hostCount: number readonly correlationId?: string + hostCount = 0 completedHosts = 0 successfulHosts = 0 emptyHosts = 0 @@ -526,199 +599,158 @@ class LookupQuerySession { rejectedHosts = 0 freeformHosts = 0 emittedFinal = false - - private readonly graceMs: number - private readonly softTimeoutMs?: number - private readonly waitForAllHosts: boolean - private readonly resolveTxId: LookupQuerySessionOptions['resolveTxId'] - private readonly outputsMap = new Map< - string, - { beef: number[]; context?: number[]; outputIndex: number } - >() + closed = false + accepting = true + discoveryComplete = false + discoveredHosts = 0 + skippedHosts = 0 + receivedBytes = 0 + retainedBytes = 0 + trackersTotal = 0 + trackersCompleted = 0 + trackersFailed = 0 + readonly limitsHit = new Set() + terminalReason: NonNullable = 'settled' + private evidenceOutputs = 0 + private evidenceBytes = 0 + private evidenceLimited = false + private limitNotificationSent = false + private readonly outputsMap = new Map() private readonly txIds: string[] = [] - private readonly queue: LookupQueryEvent[] = [] private waiter: (() => void) | null = null + private dirty = false + private finished = false + private failure: unknown private firstResponseAt: number | null = null private graceTimer: ReturnType | null = null private softTimer: ReturnType | null = null private graceFired = false - private emittedOnce = false + private softFired = false - constructor(options: LookupQuerySessionOptions) { - this.hostCount = options.hostCount - this.graceMs = options.graceMs - this.softTimeoutMs = options.softTimeoutMs - this.waitForAllHosts = options.waitForAllHosts + constructor(private readonly options: LookupQuerySessionOptions) { this.correlationId = options.correlationId - this.resolveTxId = options.resolveTxId } - private push(event: LookupQueryEvent): void { - this.queue.push(event) - if (this.waiter === null) return + wake(): void { + this.dirty = true const waiter = this.waiter this.waiter = null - waiter() + waiter?.() } - recordOutputAnswer(answer: LookupAnswer): void { - this.successfulHosts++ - if (answer.outputs.length === 0) { - this.emptyHosts++ - return + limit(name: string): void { + this.limitsHit.add(name) + if (!this.limitNotificationSent && this.accepting) { + this.limitNotificationSent = true + try { void Promise.resolve(this.options.onEvidence?.({ type: 'limit' })).catch(() => {}) } catch { /* consumer isolation */ } } - this.push({ kind: 'answer', answer }) + if (this.terminalReason === 'settled') this.terminalReason = 'resource-limit' + this.wake() } - recordFreeformAnswer(): void { - this.freeformHosts++ - } - - recordRejection(): void { - this.rejectedHosts++ + receiveEvidence(host: string, answer: LookupAnswer, callback: LookupQueryOptions['onEvidence']): void { + if (callback === undefined || this.closed || !this.accepting || this.evidenceLimited) return + const deliver = (event: LookupEvidenceEvent): void => { + try { void Promise.resolve(callback(event)).catch(() => {}) } catch { /* consumer isolation */ } + } + for (const output of answer.outputs) { + if (!this.accepting || this.closed) break + const bytes = output.beef.length + (output.context?.length ?? 0) + if (this.evidenceOutputs >= this.options.limits.maxEvidenceOutputs || + bytes > this.options.limits.maxEvidenceBytes - this.evidenceBytes) { + this.evidenceLimited = true + this.limit(this.evidenceOutputs >= this.options.limits.maxEvidenceOutputs ? 'maxEvidenceOutputs' : 'maxEvidenceBytes') + break + } + this.evidenceOutputs++ + this.evidenceBytes += bytes + deliver({ type: 'output', host, output: { + ...output, beef: output.beef.slice(), + ...(output.context === undefined ? {} : { context: output.context.slice() }) + } }) + } } - recordAvailabilityFailure(): void { - this.failedHosts++ + recordOutputAnswer(answer: LookupAnswer): void { + if (this.closed || !this.accepting) return + this.successfulHosts++ + if (answer.outputs.length === 0) { this.emptyHosts++; return } + this.mergeAnswer(answer) + if (this.firstResponseAt === null) { + this.firstResponseAt = Date.now() + if (this.options.graceMs > 0) this.graceTimer = setTimeout(() => { + this.graceFired = true; this.wake() + }, this.options.graceMs) + else this.graceFired = true + } + this.wake() } - recordDone(): void { - this.completedHosts++ - this.push({ kind: 'done' }) - } + recordFreeformAnswer(): void { if (!this.closed) this.freeformHosts++ } + recordRejection(): void { if (!this.closed) this.rejectedHosts++ } + recordAvailabilityFailure(): void { if (!this.closed) this.failedHosts++ } + recordDone(): void { if (!this.closed) { this.completedHosts++; this.wake() } } - private mergeAnswer(answer: LookupAnswer): boolean { - let added = false + private mergeAnswer(answer: LookupAnswer): void { const now = Date.now() for (const output of answer.outputs) { - const txId = this.resolveTxId(output, now) + const txId = this.options.resolveTxId(output, now) if (txId === null) continue const key = `${txId}.${output.outputIndex}` if (this.outputsMap.has(key)) continue + if (this.outputsMap.size >= this.options.limits.maxOutputs) { this.limit('maxOutputs'); break } this.outputsMap.set(key, output) this.txIds.push(txId) - added = true - } - return added - } - - snapshot(isFinal: boolean): LookupAnswerProgress { - return { - type: 'output-list', - outputs: Array.from(this.outputsMap.values()), - txIds: this.txIds.slice(), - isFinal, - hostCount: this.hostCount, - completedHosts: this.completedHosts, - successfulHosts: this.successfulHosts, - emptyHosts: this.emptyHosts, - failedHosts: this.failedHosts, - rejectedHosts: this.rejectedHosts, - freeformHosts: this.freeformHosts, - ...(this.correlationId !== undefined - ? { correlationId: this.correlationId } - : {}) } } - private handleAnswer(answer: LookupAnswer): LookupAnswerProgress | null { - const added = this.mergeAnswer(answer) - if (this.firstResponseAt === null) { - this.firstResponseAt = Date.now() - if (!this.graceFired && this.graceMs > 0) { - this.graceTimer = setTimeout(() => { - this.graceFired = true - this.push({ kind: 'grace' }) - }, this.graceMs) - } else { - this.graceFired = true - } - } - if ( - this.graceFired && - added && - (this.emittedOnce || !this.waitForAllHosts) - ) { - this.emittedOnce = true - return this.snapshot(false) - } - return null - } + finish(error?: unknown): void { this.failure = error; this.finished = true; this.wake() } - private handleGrace(): LookupAnswerProgress | null { - if (this.emittedOnce || this.waitForAllHosts) return null - this.emittedOnce = true - return this.snapshot(false) - } - - private handleSoft(): { - snapshot: LookupAnswerProgress | null - stop: boolean - } { - let snapshot: LookupAnswerProgress | null = null - if (!this.emittedOnce) { - this.graceFired = true - this.emittedOnce = true - snapshot = this.snapshot(false) - } + snapshot(isFinal: boolean): LookupAnswerProgress { return { - snapshot, - stop: - typeof this.softTimeoutMs === 'number' && - this.firstResponseAt !== null - } - } - - private async nextEvent(): Promise { - if (this.queue.length === 0) { - await new Promise(resolve => { - this.waiter = resolve - }) + type: 'output-list', outputs: Array.from(this.outputsMap.values()), txIds: this.txIds.slice(), + isFinal, hostCount: this.hostCount, completedHosts: this.completedHosts, + successfulHosts: this.successfulHosts, emptyHosts: this.emptyHosts, failedHosts: this.failedHosts, + rejectedHosts: this.rejectedHosts, freeformHosts: this.freeformHosts, + discoveryComplete: this.discoveryComplete, + ...(isFinal ? { terminalReason: this.terminalReason } : {}), + discoveredHosts: this.discoveredHosts, skippedHosts: this.skippedHosts, + receivedBytes: this.receivedBytes, retainedBytes: this.retainedBytes, evidenceBytes: this.evidenceBytes, trackersTotal: this.trackersTotal, + trackersCompleted: this.trackersCompleted, trackersFailed: this.trackersFailed, + limitsHit: Array.from(this.limitsHit), + ...(this.correlationId !== undefined ? { correlationId: this.correlationId } : {}) } - return this.queue.shift() as LookupQueryEvent } - private processEvent(event: LookupQueryEvent): { - snapshot: LookupAnswerProgress | null - stop: boolean - } { - switch (event.kind) { - case 'answer': - return { snapshot: this.handleAnswer(event.answer), stop: false } - case 'grace': - return { snapshot: this.handleGrace(), stop: false } - case 'soft': - return this.handleSoft() - case 'done': - return { snapshot: null, stop: false } - } + close(): void { + this.closed = true + this.accepting = false + if (this.graceTimer !== null) clearTimeout(this.graceTimer) + if (this.softTimer !== null) clearTimeout(this.softTimer) + this.wake() } async *progress(): AsyncIterable { - if ( - typeof this.softTimeoutMs === 'number' && - this.softTimeoutMs >= 0 - ) { - this.softTimer = setTimeout( - () => this.push({ kind: 'soft' }), - this.softTimeoutMs - ) + if (typeof this.options.softTimeoutMs === 'number' && this.options.softTimeoutMs >= 0) { + this.softTimer = setTimeout(() => { this.softFired = true; this.wake() }, this.options.softTimeoutMs) } try { - let stop = false - while (this.completedHosts < this.hostCount && !stop) { - const event = await this.nextEvent() - const outcome = this.processEvent(event) - if (outcome.snapshot != null) yield outcome.snapshot - stop = outcome.stop + while (!this.closed) { + if (this.finished) { + if (this.failure !== undefined) throw this.failure + this.emittedFinal = true + yield this.snapshot(true) + return + } + if (this.dirty && (this.softFired || (this.graceFired && !this.options.waitForAllHosts))) { + this.dirty = false + yield this.snapshot(false) + } else { + this.dirty = false + await new Promise(resolve => { this.waiter = resolve }) + } } - const finalSnapshot = this.snapshot(true) - this.emittedFinal = true - yield finalSnapshot - } finally { - if (this.graceTimer !== null) clearTimeout(this.graceTimer) - if (this.softTimer !== null) clearTimeout(this.softTimer) - } + } finally { this.close() } } } @@ -745,8 +777,11 @@ export default class LookupResolver { private readonly telemetry: Telemetry // ---- Caches / memoization ---- - private readonly hostsCache: Map - private readonly hostsInFlight: Map> + private readonly hostsCache: Map + private readonly hostsInFlight: Map + private readonly limits: LookupLimits + private activeQueries = 0 + private trackerCursor = 0 private readonly hostsTtlMs: number private readonly hostsMaxEntries: number @@ -762,6 +797,7 @@ export default class LookupResolver { private readonly lastUnreachableNotificationAt: Map constructor(config: LookupResolverConfig = {}) { + this.limits = lookupLimits(config.limits) this.networkPreset = config.networkPreset ?? 'mainnet' this.facilitator = config.facilitator ?? @@ -877,77 +913,6 @@ export default class LookupResolver { } } - private appendAdditionalHosts(service: string, hosts: string[]): void { - const additional = this.additionalHosts[service] - if (additional == null || additional.length === 0) return - const seen = new Set(hosts) - for (const host of additional) { - if (!seen.has(host)) hosts.push(host) - } - } - - private async competentHostsFor(question: LookupQuestion): Promise { - let hosts: string[] - if (question.service === 'ls_slap') { - hosts = - this.networkPreset === 'local' - ? ['http://localhost:8080'] - : this.slapTrackers - } else if (this.hostOverrides[question.service] != null) { - hosts = this.hostOverrides[question.service] - } else if (this.networkPreset === 'local') { - hosts = ['http://localhost:8080'] - } else { - hosts = await this.getCompetentHostsCached(question.service) - } - this.appendAdditionalHosts(question.service, hosts) - if (hosts.length < 1) { - throw new Error( - `No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${question.service}` - ) - } - return hosts - } - - private isSlapRecoveryEligible(service: string): boolean { - return ( - service !== 'ls_slap' && - this.hostOverrides[service] == null && - this.networkPreset !== 'local' - ) - } - - private async rankedHostsFor(question: LookupQuestion): Promise { - const competentHosts = await this.competentHostsFor(question) - let rankedHosts: string[] - try { - rankedHosts = this.prepareHostsForQuery( - competentHosts, - `lookup service ${question.service}` - ) - } catch (error) { - if (!this.isSlapRecoveryEligible(question.service)) throw error - this.hostsCache.delete(question.service) - const fresh = await this.refreshHosts(question.service, true) - this.appendAdditionalHosts(question.service, fresh) - if (fresh.length < 1) { - throw new Error( - `No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${question.service}` - ) - } - rankedHosts = this.prepareHostsForQuery( - fresh, - `lookup service ${question.service}` - ) - } - if (rankedHosts.length < 1) { - throw new Error( - `All competent hosts for ${question.service} are temporarily unavailable due to backoff.` - ) - } - return rankedHosts - } - private unreachableNotificationCooldown( options: LookupQueryOptions | undefined ): number { @@ -1059,192 +1024,292 @@ export default class LookupResolver { } } - private startLookupHostQueries( - hosts: string[], - question: LookupQuestion, - timeout: number | undefined, - session: LookupQuerySession, - options: LookupQueryOptions | undefined - ): void { - const correlationId = session.correlationId - const notificationCooldownMs = - this.unreachableNotificationCooldown(options) - for (const host of hosts) { - const hostStartedAt = Date.now() - void this.lookupHostWithTracking(host, question, timeout) - .then(answer => { - this.recordLookupHostAnswer( - session, - question.service, - host, - answer, - hostStartedAt, - correlationId - ) - }) - .catch(error => { - this.recordLookupHostFailure({ - session, - service: question.service, - host, - hostStartedAt, - correlationId, - onUnreachableHost: options?.onUnreachableHost, - notificationCooldownMs - }, error) - }) - .finally(() => { - session.recordDone() - }) + /** + * Cumulative unverified results. Discovery remains subscribed while trackers + * settle; each new host enters the bounded queue immediately. Caller abort, + * deadline and iterator close release this query's ownership. + */ + query$(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): AsyncIterable { + const cancellation = new AbortController() + const iterator = this.queryProgress(question, timeout, options, cancellation.signal)[Symbol.asyncIterator]() + return { + [Symbol.asyncIterator]: () => ({ + next: async () => await iterator.next(), + return: async () => { + cancellation.abort() + return await iterator.return?.() ?? { done: true, value: undefined } + }, + throw: async (error?: unknown) => { + cancellation.abort() + if (iterator.throw !== undefined) return await iterator.throw(error) + throw error + } + }) } } - /** - * Iterable form of {@link query}. Emits partial results as hosts answer. - * - * Emission order: - * - First emission: after the grace window expires (or as soon as the soft timeout elapses), containing - * every output gathered from hosts that answered by then. - * - Subsequent emissions: re-emitted whenever a late host returns extra outputs that weren't in earlier - * emissions. Each emission contains the cumulative `outputs` set. - * - Final emission: `isFinal: true` once all in-flight hosts have settled (success / fail / timeout). The - * caller can `break` early; outstanding work is bounded by the per-host timeout. - * - * No host work runs past its per-host `timeout` — there is no leak risk on early break. - */ - async *query$( - question: LookupQuestion, - timeout?: number, - options?: LookupQueryOptions - ): AsyncIterable { - const rankedHosts = await this.rankedHostsFor(question) - const hostCount = rankedHosts.length - const correlationId = - options?.correlationId ?? - (this.telemetry.enabled ? this.telemetry.createCorrelationId() : undefined) - const session = new LookupQuerySession({ - hostCount, - graceMs: options?.graceMs ?? 80, - softTimeoutMs: options?.softTimeoutMs, - waitForAllHosts: - options?.waitForAllHosts ?? options?.holdForUnknownHosts ?? false, - correlationId, - resolveTxId: (output, now) => this.resolveTxIdForOutput(output, now) + private async *queryProgress(inputQuestion: LookupQuestion, timeout: number | undefined, + options: LookupQueryOptions | undefined, iteratorSignal: AbortSignal): AsyncIterable { + // Capture JSON wire values once, before any discovery or queued host can + // observe a caller's later mutation. Custom non-JSON questions retain their + // historical facilitator-defined semantics when they cannot be cloned. + let question: LookupQuestion + try { question = structuredClone(inputQuestion) } + catch { + if (this.facilitator instanceof HTTPSOverlayLookupFacilitator) question = JSON.parse(stringifyBRC100(inputQuestion)) as LookupQuestion + else question = { ...inputQuestion } + } + const limits = lookupLimits(this.limits, options?.limits, { + ...(options?.evidenceLimits?.maxOutputs === undefined ? {} : { maxEvidenceOutputs: options.evidenceLimits.maxOutputs }), + ...(options?.evidenceLimits?.maxBytes === undefined ? {} : { maxEvidenceBytes: options.evidenceLimits.maxBytes }) }) - - this.telemetry.capture({ - name: 'sdk.overlay.lookup.started', - component: 'sdk.lookup-resolver', - severity: 'debug', - correlationId, - attributes: { - service: question.service, - network: this.networkPreset, - hostCount - } + const deadlineMs = options?.deadlineMs ?? 10_000 + if (!Number.isFinite(deadlineMs) || deadlineMs < 0 || deadlineMs > 2_147_483_647) { + throw new RangeError('Lookup deadlineMs must be between 0 and 2147483647') + } + if (this.activeQueries >= 128) throw new LookupResourceLimitError('activeQueries') + this.activeQueries++ + const controller = new AbortController() + const session = new LookupQuerySession({ + graceMs: options?.graceMs ?? 80, softTimeoutMs: options?.softTimeoutMs, + waitForAllHosts: options?.waitForAllHosts ?? options?.holdForUnknownHosts ?? false, + correlationId: options?.correlationId ?? (this.telemetry.enabled ? this.telemetry.createCorrelationId() : undefined), + limits, onEvidence: options?.onEvidence, resolveTxId: (output, now) => this.resolveTxIdForOutput(output, now) }) - - this.startLookupHostQueries( - rankedHosts, - question, - timeout, - session, - options - ) - - try { - for await (const progress of session.progress()) { - if (progress.isFinal) { - this.captureLookupCompletedTelemetry( - question.service, - progress, - Date.now() - session.startedAt - ) - } - yield progress + let releaseDiscovery: (() => void) | undefined + let discoveryFinished = false + let noHostsError: Error | undefined + const seen = new Set() + let sourceQuota = limits.maxHosts + let discoveryBytes = 0 + let discoverySkipped = 0 + const processedSources = new Set() + const consume = (bytes: number): void => { + if (controller.signal.aborted) throw lookupAbortError() + if (bytes > limits.maxTotalBytes - session.receivedBytes) { + session.limit('maxTotalBytes') + throw new LookupResourceLimitError('maxTotalBytes') } - } finally { - if (!session.emittedFinal) { - this.telemetry.capture({ - name: 'sdk.overlay.lookup.cancelled', - component: 'sdk.lookup-resolver', - severity: 'debug', - correlationId, - attributes: { - service: question.service, - hostCount, - completedHosts: session.completedHosts, - durationMs: Date.now() - session.startedAt - } + session.receivedBytes += bytes + } + const queue = new LookupHostQueue(limits.maxHosts, limits.hostConcurrency, async host => { + if (controller.signal.aborted) return + session.hostCount++ + const startedAt = Date.now() + try { + const answer = await this.lookupHostWithTracking(host, question, timeout, controller.signal, { + maxResponseBytes: limits.maxResponseBytes, maxOutputs: limits.maxOutputs, consumeBytes: consume }) - } + if (controller.signal.aborted || session.closed) return + let ownedAnswer = answer + if (isOutputListAnswer(answer)) { + let retained = 0 + for (const output of answer.outputs) retained += output.beef.length + (output.context?.length ?? 0) + if (retained > limits.maxTotalBytes - session.retainedBytes) throw new LookupResourceLimitError('maxTotalBytes') + session.retainedBytes += retained + ownedAnswer = { type: 'output-list', outputs: answer.outputs.map(output => ({ + ...output, beef: output.beef.slice(), ...(output.context === undefined ? {} : { context: output.context.slice() }) + })) } + session.receiveEvidence(host, ownedAnswer, options?.onEvidence) + } + if (controller.signal.aborted || session.closed) return + this.recordLookupHostAnswer(session, question.service, host, ownedAnswer, startedAt, session.correlationId) + } catch (error) { + if (controller.signal.aborted || session.closed) return + if (error instanceof LookupResourceLimitError) session.limit(error.limit) + else this.recordLookupHostFailure({ + session, service: question.service, host, hostStartedAt: startedAt, + correlationId: session.correlationId, onUnreachableHost: options?.onUnreachableHost, + notificationCooldownMs: this.unreachableNotificationCooldown(options) + }, error) + } finally { session.recordDone() } + }, (count, limited) => { if (count > 0) { session.skippedHosts += count; if (limited) session.limit('maxHosts') } }) + + const stop = (reason: 'deadline' | 'cancelled'): void => { + if (controller.signal.aborted) return + session.limit(reason) + session.terminalReason = reason + session.accepting = false + session.discoveryComplete = false + controller.abort() + releaseDiscovery?.() + discoveryFinished = true + queue.cancel() } - } - - /** - * Cached wrapper for competent host discovery with stale-while-revalidate. - */ - private async getCompetentHostsCached(service: string): Promise { - const now = Date.now() - const cached = this.hostsCache.get(service) - - // if fresh, return immediately - if (typeof cached === 'object' && cached.expiresAt > now) { - return cached.hosts.slice() + const abort = (): void => stop('cancelled') + options?.signal?.addEventListener('abort', abort, { once: true }) + iteratorSignal.addEventListener('abort', abort, { once: true }) + const timer = setTimeout(() => stop('deadline'), deadlineMs) + + let cleaned = false + const cleanup = (): void => { + if (cleaned) return + cleaned = true + session.close() + clearTimeout(timer) + options?.signal?.removeEventListener('abort', abort) + iteratorSignal.removeEventListener('abort', abort) + controller.abort() + releaseDiscovery?.() + queue.cancel() + this.activeQueries-- } - // if stale but present, kick off a refresh if not already in-flight and return stale - if (typeof cached === 'object' && cached.expiresAt <= now) { - if (!this.hostsInFlight.has(service)) { - this.hostsInFlight.set( - service, - this.refreshHosts(service).finally(() => { - this.hostsInFlight.delete(service) - }) - ) + const admit = (source: string, candidates: string[]): void => { + if (controller.signal.aborted) return + const hosts: string[] = [] + const scanLimit = Math.min(candidates.length, limits.maxHosts * 4) + if (candidates.length > scanLimit) { session.skippedHosts += candidates.length - scanLimit; session.limit('maxHosts') } + for (let candidateIndex = 0; candidateIndex < scanLimit; candidateIndex++) { + const candidate = candidates[candidateIndex] + const host = normalizeLookupHost(candidate, source === 'configured' || source === 'additional') + if (host === null) { session.skippedHosts++; continue } + if (seen.has(host)) continue + // Keep this dedup set finite, too. Source-level discovery reservations + // protect later trackers before candidates reach this queue. + if (seen.size >= limits.maxHosts) { session.skippedHosts++; session.limit('maxHosts'); continue } + seen.add(host) + session.discoveredHosts++ + hosts.push(host) } - return cached.hosts.slice() - } - - // no cache: coalesce concurrent requests - if (this.hostsInFlight.has(service)) { + if (hosts.length === 0) return try { - const hosts = await this.hostsInFlight.get(service) - if (typeof hosts !== 'object') { - throw new TypeError('Hosts is not defined.') + const available = this.prepareHostsForQuery(hosts, `lookup service ${question.service}`) + session.skippedHosts += hosts.length - available.length + queue.add(source, available) + } catch (error) { + session.skippedHosts += hosts.length + noHostsError = error instanceof Error ? error : new Error(lookupErrorMessage(error)) + } + } + const sourcesDone = (): void => { + discoveryFinished = true + queue.finishSources() + } + const acceptDiscovery = (state: LookupDiscoveryUpdate): void => { + if (controller.signal.aborted) return + session.trackersTotal = state.trackersTotal + session.trackersCompleted = state.trackersCompleted + session.trackersFailed = state.trackersFailed + session.skippedHosts += state.skippedHosts - discoverySkipped + discoverySkipped = state.skippedHosts + for (const name of state.limitsHit) session.limit(name) + try { consume(state.receivedBytes - discoveryBytes) } catch (error) { + if (error instanceof LookupResourceLimitError) session.limit(error.limit) + controller.abort() + releaseDiscovery?.() + queue.cancel() + discoveryFinished = true + return + } + discoveryBytes = state.receivedBytes + for (const [source, hosts] of state.sources) { + if (processedSources.has(source)) continue + processedSources.add(source) + admit(source, hosts.slice(0, sourceQuota)) + if (hosts.length > sourceQuota) { + session.skippedHosts += hosts.length - sourceQuota + session.limit('maxHostsPerTracker') } - return hosts.slice() - } catch { - // fall through to a fresh attempt below } + session.discoveryComplete = state.done && state.trackersFailed === 0 && state.limitsHit.size === 0 && state.skippedHosts === 0 + session.wake() + if (state.done) sourcesDone() } - - const promise = this.refreshHosts(service).finally(() => { - this.hostsInFlight.delete(service) - }) - this.hostsInFlight.set(service, promise) - const hosts = await promise - return hosts.slice() + try { + this.telemetry.capture({ name: 'sdk.overlay.lookup.started', component: 'sdk.lookup-resolver', + severity: 'debug', correlationId: session.correlationId, + attributes: { service: question.service, network: this.networkPreset, hostCount: 0 } }) + if (options?.signal?.aborted === true || iteratorSignal.aborted) abort() + if (!controller.signal.aborted) { + if (question.service === 'ls_slap' || this.hostOverrides[question.service] != null || this.networkPreset === 'local') { + const direct = question.service === 'ls_slap' + ? (this.networkPreset === 'local' ? ['http://localhost:8080'] : this.slapTrackers) + : this.hostOverrides[question.service] ?? ['http://localhost:8080'] + admit('configured', direct) + admit('additional', this.additionalHosts[question.service] ?? []) + session.discoveryComplete = true + sourcesDone() + } else { + const cached = this.hostsCache.get(question.service) + const configuredAdditional = this.additionalHosts[question.service] ?? [] + const cacheAvailable = cached?.hosts.some(host => (this.hostReputation.snapshot(host)?.backoffUntil ?? 0) <= Date.now()) ?? false + const key = JSON.stringify([question.service, limits.maxHosts, limits.maxHostsPerTracker, + limits.maxTrackers, limits.trackerConcurrency, limits.maxResponseBytes, limits.maxTotalBytes, limits.maxOutputs]) + let discovery = this.hostsInFlight.get(key) + const refresh = discovery !== undefined || cached === undefined || cached.expiresAt <= Date.now() || !cacheAvailable + const initialSources = Number(cached !== undefined && cacheAvailable) + Number(configuredAdditional.length > 0) + const initialQuota = refresh ? Math.max(1, Math.floor(limits.maxHosts / (initialSources + Math.max(1, Math.min(this.slapTrackers.length, limits.maxTrackers))))) : limits.maxHosts + if (cached !== undefined && cacheAvailable) { + // Reserve a source share for cached membership and each late tracker. + const cachedLimit = initialQuota + admit('cache', cached.hosts.slice(0, cachedLimit)) + if (cached.hosts.length > cachedLimit) { session.skippedHosts += cached.hosts.length - cachedLimit; session.limit('maxHosts') } + } + if (configuredAdditional.length > 0) { + admit('additional', configuredAdditional.slice(0, initialQuota)) + if (configuredAdditional.length > initialQuota) { session.skippedHosts += configuredAdditional.length - initialQuota; session.limit('maxHosts') } + } + if (refresh) { + sourceQuota = Math.max(1, Math.floor((limits.maxHosts - seen.size) / Math.max(1, Math.min(this.slapTrackers.length, limits.maxTrackers)))) + if (discovery === undefined) { + let trackers: string[] = [] + const selected: string[] = [] + const scan = Math.min(this.slapTrackers.length, limits.maxTrackers) + for (let i = 0; i < scan; i++) selected.push(this.slapTrackers[(this.trackerCursor + i) % this.slapTrackers.length]) + this.trackerCursor = (this.trackerCursor + scan) % Math.max(1, this.slapTrackers.length) + const normalized = Array.from(new Set(selected.map(host => normalizeLookupHost(host)).filter((host): host is string => host !== null))) + try { trackers = this.prepareHostsForQuery(normalized.slice(0, limits.maxTrackers), 'SLAP trackers') } + catch (error) { noHostsError = error instanceof Error ? error : new Error(lookupErrorMessage(error)) } + discovery = new LookupDiscovery(trackers, limits, async (tracker, signal, charge) => { + const answer = await this.lookupHostWithTracking(tracker, { service: 'ls_slap', query: { service: question.service } }, MAX_TRACKER_WAIT_TIME, signal, + { maxResponseBytes: limits.maxResponseBytes, maxOutputs: limits.maxOutputs, consumeBytes: charge }) + const hosts = isOutputListAnswer(answer) ? this.extractHostsFromAnswer(answer, question.service) : [] + for (const host of hosts) { + if (this.advertisedBy.size >= this.hostsMaxEntries * limits.maxHosts) this.evictOldest(this.advertisedBy) + this.advertisedBy.set(host, tracker) + } + return hosts + }, (state, abandoned) => { + if (this.hostsInFlight.get(key) !== discovery) return + this.hostsInFlight.delete(key) + if (abandoned) return + const hosts = Array.from(new Set(Array.from(state.sources.values()).flat())).slice(0, limits.maxHosts) + if (!this.hostsCache.has(question.service) && this.hostsCache.size >= this.hostsMaxEntries) this.evictOldest(this.hostsCache) + this.hostsCache.set(question.service, { hosts, expiresAt: Date.now() + this.hostsTtlMs, discoveryComplete: state.trackersFailed === 0 && state.limitsHit.size === 0 && state.skippedHosts === 0, trackersFailed: state.trackersFailed, limitsHit: Array.from(state.limitsHit) }) + }) + if (this.slapTrackers.length > limits.maxTrackers) discovery.state.limitsHit.add('maxTrackers') + if (normalized.length !== this.slapTrackers.length || trackers.length < Math.min(normalized.length, limits.maxTrackers)) { + discovery.state.skippedHosts += this.slapTrackers.length - trackers.length + } + this.hostsInFlight.set(key, discovery) + } + releaseDiscovery = discovery.subscribe(acceptDiscovery) + if (controller.signal.aborted) releaseDiscovery() + } else { + session.discoveryComplete = cached?.discoveryComplete ?? true + session.trackersFailed = cached?.trackersFailed ?? 0 + for (const name of cached?.limitsHit ?? []) session.limit(name) + sourcesDone() + } + } + } else queue.cancel() + void queue.done.then(() => { + const error = session.hostCount === 0 && session.terminalReason === 'settled' + ? noHostsError ?? new Error(`No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${question.service}`) + : undefined + if (discoveryFinished) session.finish(error) + }) + for await (const progress of session.progress()) { + if (progress.isFinal) { + this.captureLookupCompletedTelemetry(question.service, progress, Date.now() - session.startedAt) + cleanup() + } + yield progress + } + } finally { cleanup() } } - /** - * Actually resolves competent hosts from SLAP trackers and updates cache. - */ - private async refreshHosts( - service: string, - requireAvailable: boolean = false - ): Promise { - const hosts = await this.findCompetentHosts(service, requireAvailable) - const expiresAt = Date.now() + this.hostsTtlMs - - // bounded cache with simple FIFO eviction - if (!this.hostsCache.has(service) && this.hostsCache.size >= this.hostsMaxEntries) { - const oldestKey = this.hostsCache.keys().next().value - if (oldestKey !== undefined) this.hostsCache.delete(oldestKey) - } - this.hostsCache.set(service, { hosts, expiresAt }) - return hosts - } /** * Extracts competent host domains from a SLAP tracker response. @@ -1269,70 +1334,6 @@ export default class LookupResolver { return hosts } - /** - * Returns a list of competent hosts for a given lookup service. - * Resolves as soon as the first SLAP tracker responds with valid hosts. - * Remaining trackers continue in the background for reputation tracking. - * @param service Service for which competent hosts are to be returned - * @returns Array of hosts competent for resolving queries - */ - private async findCompetentHosts( - service: string, - requireAvailable: boolean = false - ): Promise { - const query: LookupQuestion = { - service: 'ls_slap', - query: { service } - } - - const trackerHosts = this.prepareHostsForQuery(this.slapTrackers, 'SLAP trackers') - if (trackerHosts.length === 0) return [] - - // Fire all trackers, resolve as soon as any returns valid hosts. - // Remaining trackers continue in the background for reputation tracking. - return await new Promise(resolve => { - const allHosts = new Set() - let resolved = false - let pending = trackerHosts.length - - for (const tracker of trackerHosts) { - this.lookupHostWithTracking(tracker, query, MAX_TRACKER_WAIT_TIME) - .then(answer => { - const hosts = isOutputListAnswer(answer) - ? this.extractHostsFromAnswer(answer, service) - : [] - for (const h of hosts) { - if (!allHosts.has(h)) { - allHosts.add(h) - // First-seen attribution: the tracker that surfaced this host - // gets credit, used by onUnreachableHost callbacks. - this.advertisedBy.set(h, tracker) - } - } - const now = Date.now() - const foundAvailable = [...allHosts].some(host => { - const backoffUntil = this.hostReputation.snapshot(host)?.backoffUntil ?? 0 - return backoffUntil <= now - }) - if (!resolved && allHosts.size > 0 && (!requireAvailable || foundAvailable)) { - resolved = true - resolve([...allHosts]) - } - }) - .catch(() => { - /* tracker failure tracked in reputation */ - }) - .finally(() => { - pending-- - if (pending === 0 && !resolved) { - resolved = true - resolve([...allHosts]) - } - }) - } - }) - } - /** * Resolve a txid for an aggregated lookup output. Uses the threaded-through `output.txid` * fast path when present; otherwise memoizes Transaction.fromBEEF(beef).id('hex') keyed by @@ -1345,7 +1346,7 @@ export default class LookupResolver { if (typeof output.txid === 'string' && output.txid.length > 0) { return output.txid } - const keyForBeef = Array.isArray(output.beef) ? output.beef.join(',') : '' + const keyForBeef = Utils.toHex(sha256(output.beef)) const memo = this.txMemo.get(keyForBeef) if (typeof memo === 'object' && memo !== null && memo.expiresAt > now) { return memo.txId @@ -1391,18 +1392,27 @@ export default class LookupResolver { private async lookupHostWithTracking( host: string, question: LookupQuestion, - timeout?: number + timeout?: number, + signal?: AbortSignal, + options?: LookupRequestOptions ): Promise { const startedAt = Date.now() const effectiveTimeout = typeof timeout === 'number' && Number.isFinite(timeout) && timeout >= 0 ? timeout : DEFAULT_LOOKUP_TIMEOUT - const deadline = createDeadline(effectiveTimeout) + const controller = new AbortController() + const abort = (): void => controller.abort() + signal?.addEventListener('abort', abort, { once: true }) + const deadline = createDeadline(effectiveTimeout, controller) + let reportedBytes = 0 + const requestOptions = { ...options, consumeBytes: (bytes: number): void => { + options?.consumeBytes?.(bytes); reportedBytes += bytes + } } // Start the custom facilitator in a promise chain so synchronous throws // become rejections governed by the same wall-clock deadline. const lookupPromise = Promise.resolve().then(() => - this.facilitator.lookup(host, question, timeout) + signal?.aborted === true ? Promise.reject(lookupAbortError()) : this.facilitator.lookup(host, question, timeout, controller.signal, requestOptions) ) lookupPromise.catch(() => { /* deadline may win while custom facilitator settles later */ @@ -1410,16 +1420,28 @@ export default class LookupResolver { let answer: LookupFacilitatorAnswer try { - answer = await Promise.race([lookupPromise, deadline.promise]) + answer = await withLookupAbort(Promise.race([lookupPromise, deadline.promise]), signal) } catch (err) { + if (signal?.aborted === true) throw lookupAbortError() + if (err instanceof LookupResourceLimitError) throw err const normalized = normalizeLookupError(err, deadline.didTimeOut()) if (!isSemanticLookupRejection(err)) this.hostReputation.recordFailure(host, normalized) throw isSemanticLookupRejection(err) ? err : normalized } finally { deadline.cancel() + signal?.removeEventListener('abort', abort) + controller.abort() } + if (signal?.aborted === true) throw lookupAbortError() + if (answer != null && answer.type === 'output-list' && Array.isArray(answer.outputs) && answer.outputs.length > (options?.maxOutputs ?? DEFAULT_LOOKUP_LIMITS.maxOutputs)) throw new LookupResourceLimitError('maxOutputs') if (isOutputListAnswer(answer)) { + let bytes = 0 + for (const output of answer.outputs) { + bytes += output.beef.length + (output.context?.length ?? 0) + if (bytes > (options?.maxResponseBytes ?? DEFAULT_LOOKUP_LIMITS.maxResponseBytes)) throw new LookupResourceLimitError('maxResponseBytes') + } + if (reportedBytes === 0) options?.consumeBytes?.(bytes) this.hostReputation.recordSuccess(host, Date.now() - startedAt) return answer } diff --git a/packages/sdk/src/overlay-tools/LookupResources.ts b/packages/sdk/src/overlay-tools/LookupResources.ts new file mode 100644 index 000000000..8fee3a773 --- /dev/null +++ b/packages/sdk/src/overlay-tools/LookupResources.ts @@ -0,0 +1,83 @@ +/** Operational client limits, not BEEF validity or service authority rules. */ +export interface LookupLimits { + maxHosts: number + maxHostsPerTracker: number + maxTrackers: number + hostConcurrency: number + trackerConcurrency: number + maxResponseBytes: number + maxTotalBytes: number + maxOutputs: number + maxEvidenceOutputs: number + maxEvidenceBytes: number +} + +/** Finite defaults; applications with larger proofs can raise these explicitly. */ +export const DEFAULT_LOOKUP_LIMITS: Readonly = Object.freeze({ + maxHosts: 256, + maxHostsPerTracker: 64, + maxTrackers: 16, + hostConcurrency: 8, + trackerConcurrency: 4, + maxResponseBytes: 32 * 1024 * 1024, + maxTotalBytes: 64 * 1024 * 1024, + maxOutputs: 4096, + maxEvidenceOutputs: 512, + maxEvidenceBytes: 16 * 1024 * 1024 +}) + +export class LookupResourceLimitError extends Error { + constructor(readonly limit: string) { + super(`Lookup resource limit reached: ${limit}`) + this.name = 'LookupResourceLimitError' + } +} + +export function lookupLimits(...overrides: Array | undefined>): LookupLimits { + const limits = Object.assign({}, DEFAULT_LOOKUP_LIMITS, ...overrides) + for (const [name, value] of Object.entries(limits)) { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) { + throw new RangeError(`Lookup limit ${name} must be a positive safe integer`) + } + } + return limits +} + +/** Preserve distinct paths and ports; remove only a final slash and URL fragments. */ +export function normalizeLookupHost(host: string, allowParameters: boolean = false): string | null { + if (typeof host !== 'string' || host.length > 2048) return null + try { + const url = new URL(host) + if (!['https:', 'http:'].includes(url.protocol) || url.username !== '' || url.password !== '') return null + // A query/fragment has no defined meaning before the /lookup route suffix. + if (!allowParameters && (url.search !== '' || url.hash !== '')) return null + return url.href.replace(/\/$/, '') + } catch { + return null + } +} + +export function lookupAbortError(): Error { + const error = new Error('Lookup cancelled') + error.name = 'AbortError' + return error +} + +/** A non-cooperative transport cannot retain a cancelled waiter. */ +export async function withLookupAbort(work: Promise, signal?: AbortSignal): Promise { + if (signal === undefined) return await work + if (signal.aborted) { + void work.catch(() => {}) + throw lookupAbortError() + } + let abort = (): void => {} + const cancelled = new Promise((_resolve, reject) => { + abort = () => reject(lookupAbortError()) + signal.addEventListener('abort', abort, { once: true }) + }) + try { + return await Promise.race([work, cancelled]) + } finally { + signal.removeEventListener('abort', abort) + } +} diff --git a/packages/sdk/src/overlay-tools/LookupResponseReader.ts b/packages/sdk/src/overlay-tools/LookupResponseReader.ts new file mode 100644 index 000000000..fe0ed7b4b --- /dev/null +++ b/packages/sdk/src/overlay-tools/LookupResponseReader.ts @@ -0,0 +1,189 @@ +import { LookupResourceLimitError } from './LookupResources.js' + +/** Options controlling a bounded lookup response read. */ +export interface LookupResponseReaderOptions { + /** Cancels a pending stream read when the lookup request is aborted. */ + signal?: AbortSignal + /** Maximum number of response bytes to retain. */ + maxResponseBytes: number + /** Charges accepted bytes to the caller's aggregate response budget. */ + consumeBytes?: (bytes: number) => void +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException('The operation was aborted.', 'AbortError') +} + +function assertValidMaximum(maxResponseBytes: number): void { + if (!Number.isSafeInteger(maxResponseBytes) || maxResponseBytes < 0) { + throw new RangeError('maxResponseBytes must be a non-negative safe integer') + } +} + +function assertDeclaredLengthIsWithinLimit(response: Response, maxResponseBytes: number): void { + const contentLength = response.headers.get('content-length') + if (contentLength === null) return + + const normalized = contentLength.trim() + // Content-Length is decimal bytes. Treat malformed fields as unknown rather + // than accidentally accepting a notation such as "1e6". + if (!/^[0-9]+$/.test(normalized)) return + + const declaredLength = Number(normalized) + if (!Number.isSafeInteger(declaredLength) || declaredLength > maxResponseBytes) { + throw new LookupResourceLimitError('maxResponseBytes') + } +} + +async function readWithAbort( + reader: ReadableStreamDefaultReader, + signal: AbortSignal | undefined +): Promise> { + if (signal === undefined) return await reader.read() + if (signal.aborted) throw abortReason(signal) + + return await new Promise>((resolve, reject) => { + let settled = false + const finish = (callback: () => void): void => { + if (settled) return + settled = true + signal.removeEventListener('abort', onAbort) + callback() + } + const onAbort = (): void => finish(() => reject(abortReason(signal))) + + signal.addEventListener('abort', onAbort, { once: true }) + try { + Promise.resolve(reader.read()).then( + result => finish(() => resolve(result)), + error => finish(() => reject(error)) + ) + } catch (error) { + finish(() => reject(error)) + } + + // Do not miss an abort that happened while registering the listener. + if (signal.aborted) onAbort() + }) +} + +function cleanUpFailedRead(reader: ReadableStreamDefaultReader, reason: unknown): void { + try { + void Promise.resolve(reader.cancel(reason)).catch(() => undefined) + } catch { + // A broken stream implementation must not replace the response read error. + } + + try { + reader.releaseLock() + } catch { + // The lock may already have been released by a nonstandard stream. + } +} + +function expandedBuffer( + buffer: Uint8Array, + requiredLength: number, + maxResponseBytes: number +): Uint8Array { + if (requiredLength <= buffer.byteLength) return buffer + + const initialCapacity = Math.min(maxResponseBytes, 1024) + const doubledCapacity = Math.min(maxResponseBytes, buffer.byteLength * 2) + const capacity = Math.max( + requiredLength, + buffer.byteLength === 0 ? initialCapacity : doubledCapacity + ) + const expanded = new Uint8Array(capacity) + expanded.set(buffer) + return expanded +} + +function yieldToEventLoop(): Promise { + return new Promise(resolve => setTimeout(resolve, 0)) +} + +async function yieldAfterReadIfNeeded( + readOperations: number, + signal: AbortSignal | undefined +): Promise { + if (readOperations % 64 !== 0) return + await yieldToEventLoop() + if (signal?.aborted) throw abortReason(signal) +} + +/** + * Reads a lookup response incrementally while enforcing a per-response bound. + * + * This deliberately does not use Response.text(), json(), or arrayBuffer(), + * because those APIs buffer the complete body before a limit can be enforced. + */ +export async function readLookupResponseBytes( + response: Response, + options: LookupResponseReaderOptions +): Promise { + const { signal, maxResponseBytes, consumeBytes } = options + assertValidMaximum(maxResponseBytes) + + const body = response.body + if (body === null) { + assertDeclaredLengthIsWithinLimit(response, maxResponseBytes) + if (signal?.aborted === true) throw abortReason(signal) + return new Uint8Array(0) + } + + const reader = body.getReader() + let succeeded = false + let failure: unknown + try { + assertDeclaredLengthIsWithinLimit(response, maxResponseBytes) + if (signal?.aborted === true) throw abortReason(signal) + + let bytes: Uint8Array = new Uint8Array(0) + let totalLength = 0 + let readOperations = 0 + while (true) { + const { done, value } = await readWithAbort(reader, signal) + readOperations++ + if (done) break + + if (value.byteLength === 0) { + // An eagerly fulfilled read() still schedules only microtasks. Yielding + // periodically lets timers deliver cancellation for endless empty input. + await yieldAfterReadIfNeeded(readOperations, signal) + continue + } + + if (value.byteLength > maxResponseBytes - totalLength) { + throw new LookupResourceLimitError('maxResponseBytes') + } + + consumeBytes?.(value.byteLength) + const nextLength = totalLength + value.byteLength + bytes = expandedBuffer(bytes, nextLength, maxResponseBytes) + // Streams are allowed to reuse a producer-owned Uint8Array. Copy each + // accepted chunk now instead of retaining a mutable producer reference. + bytes.set(value, totalLength) + totalLength = nextLength + // Copy before yielding: a producer may reuse or mutate its buffer while + // the task queue runs. + await yieldAfterReadIfNeeded(readOperations, signal) + } + + succeeded = true + return bytes.subarray(0, totalLength) + } catch (error) { + failure = error + throw error + } finally { + if (succeeded) { + try { + reader.releaseLock() + } catch { + // A nonstandard stream may have released its lock itself. + } + } else { + cleanUpFailedRead(reader, failure) + } + } +} diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts index 02aff25a4..358dbba5f 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts @@ -15,6 +15,15 @@ const mockFacilitator = { lookup: jest.fn() } +const jsonResponse = (body: unknown, status = 200): Response => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' } + }) + +const octetResponse = (payload: Uint8Array): Response => + new Response(payload, { headers: { 'content-type': 'application/octet-stream' } }) + // -------------------------------------------------------------------------- // Sample BEEFs for use in tests // -------------------------------------------------------------------------- @@ -469,11 +478,9 @@ describe('LookupResolver – additional coverage', () => { }) it('allows HTTP URLs when allowHTTP is true', async () => { - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/json' }, - json: async () => ({ type: 'output-list', outputs: [] }) - }) + const mockFetch = jest + .fn() + .mockResolvedValue(jsonResponse({ type: 'output-list', outputs: [] })) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) const result = await facilitator.lookup('http://localhost:8080', { service: 'ls_test', @@ -483,12 +490,7 @@ describe('LookupResolver – additional coverage', () => { }) it('handles HTTP error responses by throwing', async () => { - const mockFetch = jest.fn().mockResolvedValue({ - ok: false, - status: 503, - headers: { get: () => 'application/json' }, - json: async () => ({}) - }) + const mockFetch = jest.fn().mockResolvedValue(jsonResponse({}, 503)) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) await expect( facilitator.lookup('http://host', { service: 'ls_test', query: {} }) @@ -544,12 +546,7 @@ describe('LookupResolver – additional coverage', () => { const beefBuf = Buffer.from(beef) const payload = Buffer.concat([nOutpoints, txid, outputIndex, contextLen, beefBuf]) - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/octet-stream' }, - arrayBuffer: async () => - payload.buffer.slice(payload.byteOffset, payload.byteOffset + payload.byteLength) - }) + const mockFetch = jest.fn().mockResolvedValue(octetResponse(payload)) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) const result = await facilitator.lookup('http://host', { service: 'ls_test', query: {} }) @@ -581,12 +578,9 @@ describe('LookupResolver – additional coverage', () => { 'Application/Octet-Stream', ' application/octet-stream ' ]) { - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => header }, - arrayBuffer: async () => - payload.buffer.slice(payload.byteOffset, payload.byteOffset + payload.byteLength) - }) + const mockFetch = jest + .fn() + .mockResolvedValue(new Response(payload, { headers: { 'content-type': header } })) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) const result = await facilitator.lookup('https://host', { service: 'ls_test', query: {} }) expect(result.type).toBe('output-list') @@ -620,12 +614,7 @@ describe('LookupResolver – additional coverage', () => { beefBuf ]) - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/octet-stream' }, - arrayBuffer: async () => - payload.buffer.slice(payload.byteOffset, payload.byteOffset + payload.byteLength) - }) + const mockFetch = jest.fn().mockResolvedValue(octetResponse(payload)) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) const result = await facilitator.lookup('http://host', { service: 'ls_test', query: {} }) @@ -688,11 +677,9 @@ describe('LookupResolver – additional coverage', () => { }) it('sends correct request body to /lookup endpoint', async () => { - const mockFetch = jest.fn().mockResolvedValue({ - ok: true, - headers: { get: () => 'application/json' }, - json: async () => ({ type: 'output-list', outputs: [] }) - }) + const mockFetch = jest + .fn() + .mockResolvedValue(jsonResponse({ type: 'output-list', outputs: [] })) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) const question = { service: 'ls_test', query: { filter: 'abc' } } await facilitator.lookup('http://host', question) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts new file mode 100644 index 000000000..a8a78f552 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts @@ -0,0 +1,841 @@ +import LookupResolver, { LookupAnswerProgress } from '../LookupResolver' +import { getOverlayHostReputationTracker } from '../HostReputationTracker' +import OverlayAdminTokenTemplate from '../OverlayAdminTokenTemplate' +import { CompletedProtoWallet } from '../../auth/certificates/__tests/CompletedProtoWallet' +import { PrivateKey } from '../../primitives/index' +import { LockingScript } from '../../script/index' +import { Transaction } from '../../transaction/index' + +const makeBeef = (satoshis: number): number[] => + new Transaction(1, [], [{ lockingScript: LockingScript.fromHex('88'), satoshis }], 0).toBEEF() + +const later = async (ms: number): Promise => { + await new Promise(resolve => setTimeout(resolve, ms)) +} + +async function overlayReceipt( + protocol: 'SHIP' | 'SLAP', + scalar: number, + domain: string, + topicOrService: string +): Promise<{ beef: number[]; outputIndex: number }> { + const wallet = new CompletedProtoWallet(new PrivateKey(scalar)) + const token = new OverlayAdminTokenTemplate(wallet) + const lockingScript = await token.lock(protocol, domain, topicOrService) + const transaction = new Transaction(1, [], [{ lockingScript, satoshis: 1 }], 0) + return { beef: transaction.toBEEF(), outputIndex: 0 } +} + +async function slapReceipt( + scalar: number, + domain: string, + service: string +): Promise<{ beef: number[]; outputIndex: number }> { + return await overlayReceipt('SLAP', scalar, domain, service) +} + +describe('LookupResolver dynamic discovery', () => { + beforeEach(() => { + getOverlayHostReputationTracker().reset() + jest.useFakeTimers() + }) + + afterEach(() => { + jest.useRealTimers() + }) + + it('starts a discovered host without waiting for a slower tracker and merges only the late host contribution', async () => { + const fastTracker = 'https://fast-tracker.example' + const slowTracker = 'https://slow-tracker.example' + const fastHost = 'https://fast-host.example' + const lateHost = 'https://late-host.example' + const service = 'ls_dynamic' + const fastReceipt = await slapReceipt(101, fastHost, service) + const lateReceipt = await slapReceipt(102, lateHost, service) + const fastBeef = makeBeef(1) + const lateBeef = makeBeef(2) + const calls: Array<{ url: string; service: string }> = [] + const evidence: Array<{ type: string; host?: string }> = [] + + const lookup = jest.fn(async (url: string, question: { service: string }) => { + calls.push({ url, service: question.service }) + if (url === fastTracker) { + await later(10) + return { type: 'output-list' as const, outputs: [fastReceipt] } + } + if (url === slowTracker) { + await later(100) + return { type: 'output-list' as const, outputs: [lateReceipt] } + } + if (url === fastHost) { + await later(1) + return { type: 'output-list' as const, outputs: [{ beef: fastBeef, outputIndex: 0 }] } + } + if (url === lateHost) { + await later(1) + return { type: 'output-list' as const, outputs: [{ beef: lateBeef, outputIndex: 1 }] } + } + throw new Error(`unexpected host ${url}`) + }) + const resolver = new LookupResolver({ + facilitator: { lookup }, + slapTrackers: [fastTracker, slowTracker] + }) + const progress: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service, query: { q: 1 } }, undefined, { + graceMs: 0, + onEvidence: event => evidence.push(event) + })) { + progress.push({ ...item, outputs: item.outputs.slice() }) + } + })() + + await jest.advanceTimersByTimeAsync(20) + expect(calls).toContainEqual({ url: fastHost, service }) + expect(calls).not.toContainEqual({ url: lateHost, service }) + expect(calls.filter(call => call.url === fastTracker || call.url === slowTracker)).toEqual([ + { url: fastTracker, service: 'ls_slap' }, + { url: slowTracker, service: 'ls_slap' } + ]) + expect(evidence).toEqual( + expect.arrayContaining([expect.objectContaining({ type: 'output', host: fastHost })]) + ) + + await jest.advanceTimersByTimeAsync(200) + await pending + + const final = progress.at(-1) + expect(final?.isFinal).toBe(true) + expect(final?.outputs).toEqual([ + { beef: fastBeef, outputIndex: 0 }, + { beef: lateBeef, outputIndex: 1 } + ]) + expect(final).toMatchObject({ discoveryComplete: true, trackersTotal: 2, trackersCompleted: 2 }) + // The SLAP receipt is discovery evidence only; it never becomes a topic result. + expect( + final?.outputs.some( + output => output.beef === fastReceipt.beef || output.beef === lateReceipt.beef + ) + ).toBe(false) + }) + + it('queries only eligible advertised hosts for the requested service', async () => { + const fooTracker = 'https://foo-tracker.example' + const barTracker = 'https://bar-tracker.example' + const shipTracker = 'https://ship-tracker.example' + const offlineTracker = 'https://offline-tracker.example' + const fooHost = 'https://foo-host.example' + const barHost = 'https://bar-host.example' + const shipHost = 'https://ship-host.example' + const service = 'ls_foo' + const fooReceipt = await slapReceipt(201, fooHost, service) + const barReceipt = await slapReceipt(202, barHost, 'ls_bar') + const shipReceipt = await overlayReceipt('SHIP', 203, shipHost, service) + const hostCalls: string[] = [] + const lookup = jest.fn(async (url: string, question: { service: string }) => { + if (url === fooTracker) return { type: 'output-list' as const, outputs: [fooReceipt] } + if (url === barTracker) return { type: 'output-list' as const, outputs: [barReceipt] } + if (url === shipTracker) return { type: 'output-list' as const, outputs: [shipReceipt] } + if (url === offlineTracker) throw new Error('tracker offline') + hostCalls.push(url) + expect(question.service).toBe(service) + return { type: 'output-list' as const, outputs: [{ beef: makeBeef(9), outputIndex: 0 }] } + }) + const resolver = new LookupResolver({ + facilitator: { lookup }, + slapTrackers: [fooTracker, barTracker, shipTracker, offlineTracker] + }) + const pending = resolver.query({ service, query: {} }) + await jest.runAllTimersAsync() + await expect(pending).resolves.toEqual({ + type: 'output-list', + outputs: [{ beef: makeBeef(9), outputIndex: 0 }] + }) + + expect(hostCalls).toEqual([fooHost]) + expect(lookup.mock.calls.map(([url]) => url)).toEqual( + expect.arrayContaining([fooTracker, barTracker, shipTracker, offlineTracker]) + ) + }) + + it('delivers a useful host while another peer never finishes, then settles after the 2s host bound', async () => { + const usefulHost = 'https://useful-hang.example' + const hangingHost = 'https://hanging-peer.example' + const usefulBeef = makeBeef(11) + const lookup = jest.fn(async (url: string, _question: unknown, timeout?: number) => { + expect(timeout).toBeUndefined() + if (url === hangingHost) await new Promise(() => {}) + await later(20) + return { type: 'output-list' as const, outputs: [{ beef: usefulBeef, outputIndex: 0 }] } + }) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_hang: [usefulHost, hangingHost] } + }) + const progress: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_hang', query: {} }, undefined, { + graceMs: 0 + })) { + progress.push({ ...item, outputs: item.outputs.slice() }) + } + })() + + await jest.advanceTimersByTimeAsync(100) + expect(progress.some(item => !item.isFinal && item.outputs.length > 0)).toBe(true) + expect(progress.find(item => item.outputs.length > 0)?.outputs).toEqual([ + { beef: usefulBeef, outputIndex: 0 } + ]) + expect(progress.at(-1)?.isFinal).toBe(false) + + await jest.advanceTimersByTimeAsync(2000) + await pending + expect(progress.at(-1)).toMatchObject({ + isFinal: true, + terminalReason: 'settled', + successfulHosts: 1, + failedHosts: 1, + outputs: [{ beef: usefulBeef, outputIndex: 0 }] + }) + }) + + it('unblocks a pending iterator once on abort and reports a cancelled terminal snapshot', async () => { + const host = 'https://pending.example' + const controller = new AbortController() + let requestSignal: AbortSignal | undefined + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + requestSignal = signal + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_abort: [host] } + }) + const iterator = resolver + .query$({ service: 'ls_abort', query: {} }, undefined, { + signal: controller.signal + } as any) + [Symbol.asyncIterator]() + + const first = iterator.next() + await Promise.resolve() + controller.abort(new Error('caller stopped lookup')) + + const terminal = await first + expect(requestSignal?.aborted).toBe(true) + expect(terminal.done).toBe(false) + expect(terminal.value).toMatchObject({ + isFinal: true, + terminalReason: 'cancelled' + }) + await expect(iterator.next()).resolves.toEqual({ done: true, value: undefined }) + }) + + it('emits a deadline terminal snapshot when no host receipt arrives', async () => { + const host = 'https://deadline.example' + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_deadline: [host] } + }) + const received: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_deadline', query: {} }, undefined, { + deadlineMs: 25 + } as any)) { + received.push(item) + } + })() + + await jest.advanceTimersByTimeAsync(25) + await pending + + expect(received).toHaveLength(1) + expect(received[0]).toMatchObject({ + isFinal: true, + terminalReason: 'deadline', + outputs: [] + }) + }) + + it('keeps empty, failed, and freeform host receipts distinct at a settled terminal', async () => { + const emptyHost = 'https://empty.example' + const freeformHost = 'https://freeform.example' + const failedHost = 'https://failed.example' + const lookup = jest.fn(async (url: string) => { + if (url === emptyHost) return { type: 'output-list' as const, outputs: [] } + if (url === freeformHost) return { type: 'freeform' as const, result: { supported: false } } + throw new Error('offline') + }) + const resolver = new LookupResolver({ + facilitator: { lookup }, + hostOverrides: { ls_outcomes: [emptyHost, freeformHost, failedHost] } + }) + const values: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_outcomes', query: {} })) + values.push(item) + })() + await jest.runAllTimersAsync() + await pending + + expect(values).toHaveLength(1) + expect(values[0]).toMatchObject({ + isFinal: true, + terminalReason: 'settled', + emptyHosts: 1, + freeformHosts: 1, + failedHosts: 1, + rejectedHosts: 0 + }) + }) + + it('caps evidence delivery independently and reports the limit evidence', async () => { + const host = 'https://bounded.example' + const first = makeBeef(31) + const second = makeBeef(32) + const evidence: Array<{ type: string }> = [] + const resolver = new LookupResolver({ + facilitator: { + lookup: async () => ({ + type: 'output-list', + outputs: [ + { beef: first, outputIndex: 0 }, + { beef: second, outputIndex: 1 } + ] + }) + }, + hostOverrides: { ls_bounded: [host] } + }) + const values: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_bounded', query: {} }, undefined, { + limits: { maxEvidenceOutputs: 1 }, + onEvidence: event => evidence.push(event) + } as any)) { + values.push(item) + } + })() + await jest.runAllTimersAsync() + await pending + + expect(values.at(-1)).toMatchObject({ + isFinal: true, + terminalReason: 'resource-limit', + limitsHit: expect.arrayContaining(['maxEvidenceOutputs']) + }) + expect(values.at(-1)?.outputs).toHaveLength(2) + expect(evidence).toEqual(expect.arrayContaining([expect.objectContaining({ type: 'limit' })])) + }) + + it('shares one tracker discovery between queries and keeps it alive when one subscriber cancels', async () => { + const tracker = 'https://shared-tracker.example' + const host = 'https://shared-host.example' + const receipt = await slapReceipt(103, host, 'ls_shared') + let completeTracker: (() => void) | undefined + let trackerSignal: AbortSignal | undefined + const lookup = jest.fn( + async (url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => { + if (url === tracker) { + trackerSignal = signal + await new Promise(resolve => { + completeTracker = resolve + }) + return { type: 'output-list' as const, outputs: [receipt] } + } + return { type: 'output-list' as const, outputs: [{ beef: makeBeef(41), outputIndex: 0 }] } + } + ) + const resolver = new LookupResolver({ facilitator: { lookup } as any, slapTrackers: [tracker] }) + const firstAbort = new AbortController() + const first = resolver + .query$({ service: 'ls_shared', query: { caller: 1 } }, undefined, { + signal: firstAbort.signal + } as any) + [Symbol.asyncIterator]() + const secondProgress: LookupAnswerProgress[] = [] + const second = (async () => { + for await (const progress of resolver.query$({ + service: 'ls_shared', + query: { caller: 2 } + })) { + secondProgress.push(progress) + } + })() + + const firstPending = first.next() + await Promise.resolve() + firstAbort.abort() + await expect(firstPending).resolves.toMatchObject({ value: { terminalReason: 'cancelled' } }) + expect(trackerSignal?.aborted).toBe(false) + completeTracker?.() + await jest.runAllTimersAsync() + await second + expect(secondProgress.at(-1)).toMatchObject({ + isFinal: true, + terminalReason: 'settled', + outputs: [{ beef: makeBeef(41), outputIndex: 0 }] + }) + expect(lookup.mock.calls.filter(([url]) => url === tracker)).toHaveLength(1) + }) + + it('does not let an abandoned custom-facilitator completion leak into a later query or its evidence', async () => { + const host = 'https://late-custom.example' + const staleBeef = makeBeef(51) + const freshBeef = makeBeef(52) + let resolveStale: + | ((value: { + type: 'output-list' + outputs: Array<{ beef: number[]; outputIndex: number }> + }) => void) + | undefined + let calls = 0 + const lookup = jest.fn(() => { + calls++ + if (calls === 1) { + return new Promise<{ + type: 'output-list' + outputs: Array<{ beef: number[]; outputIndex: number }> + }>(resolve => { + resolveStale = resolve + }) + } + return Promise.resolve({ + type: 'output-list' as const, + outputs: [{ beef: freshBeef, outputIndex: 0 }] + }) + }) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_late_custom: [host] } + }) + const abort = new AbortController() + const abandonedEvidence: Array<{ type: string }> = [] + const abandoned = resolver + .query$({ service: 'ls_late_custom', query: { generation: 1 } }, undefined, { + signal: abort.signal, + onEvidence: event => abandonedEvidence.push(event) + } as any) + [Symbol.asyncIterator]() + const abandonedPending = abandoned.next() + await Promise.resolve() + abort.abort() + await abandonedPending + + const next = resolver.query({ service: 'ls_late_custom', query: { generation: 2 } }) + resolveStale?.({ type: 'output-list', outputs: [{ beef: staleBeef, outputIndex: 0 }] }) + await jest.runAllTimersAsync() + await expect(next).resolves.toEqual({ + type: 'output-list', + outputs: [{ beef: freshBeef, outputIndex: 0 }] + }) + expect(abandonedEvidence).toEqual([{ type: 'limit' }]) + }) + + it('stops evidence and aggregate intake when the first evidence callback aborts', async () => { + const host = 'https://callback-abort.example' + const controller = new AbortController() + const events: Array<{ type: string; output?: unknown }> = [] + const resolver = new LookupResolver({ + facilitator: { + lookup: async () => ({ + type: 'output-list' as const, + outputs: [ + { beef: makeBeef(201), outputIndex: 0 }, + { beef: makeBeef(202), outputIndex: 1 }, + { beef: makeBeef(203), outputIndex: 2 } + ] + }) + }, + hostOverrides: { ls_callback_abort: [host] } + }) + const iterator = resolver + .query$({ service: 'ls_callback_abort', query: {} }, undefined, { + signal: controller.signal, + onEvidence: event => { + events.push(event) + if (event.type === 'output') controller.abort() + } + }) + [Symbol.asyncIterator]() + + const terminal = await iterator.next() + expect(terminal.value).toMatchObject({ + isFinal: true, + terminalReason: 'cancelled', + outputs: [] + }) + expect(events.filter(event => event.type === 'output')).toHaveLength(1) + expect(events.filter(event => event.type === 'limit')).toHaveLength(1) + await expect(iterator.next()).resolves.toEqual({ done: true, value: undefined }) + }) + + it('aborts and settles a pending iterator when return() is called', async () => { + const host = 'https://return-pending.example' + let requestSignal: AbortSignal | undefined + const resolver = new LookupResolver({ + facilitator: { + lookup: async (_url, _question, _timeout, signal) => + await new Promise((_resolve, reject) => { + requestSignal = signal + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + }, + hostOverrides: { ls_return_pending: [host] } + }) + const iterator = resolver + .query$({ service: 'ls_return_pending', query: {} }) + [Symbol.asyncIterator]() + const pending = iterator.next() + await Promise.resolve() + await expect(iterator.return?.()).resolves.toEqual({ done: true, value: undefined }) + expect(requestSignal?.aborted).toBe(true) + await expect(pending).resolves.toMatchObject({ + done: false, + value: { isFinal: true, terminalReason: 'cancelled' } + }) + }) + + it('cleans terminal queries before yielding so 130 final-only consumers do not exhaust slots', async () => { + const resolver = new LookupResolver({ + facilitator: { lookup: async () => ({ type: 'output-list' as const, outputs: [] }) }, + hostOverrides: { ls_final_only: ['https://final-only.example'] } + }) + for (let index = 0; index < 130; index++) { + const first = await resolver + .query$({ service: 'ls_final_only', query: { index } }) + [Symbol.asyncIterator]() + .next() + expect(first.value?.isFinal).toBe(true) + } + expect((resolver as any).activeQueries).toBe(0) + }) + + it('enforces decoded BEEF and context retention across hosts independently of reported wire bytes', async () => { + const firstHost = 'https://decoded-one.example' + const secondHost = 'https://decoded-two.example' + const firstOutput = { beef: makeBeef(211), outputIndex: 0, context: Array(10).fill(7) } + const secondOutput = { beef: makeBeef(212), outputIndex: 1, context: Array(10).fill(8) } + const resolver = new LookupResolver({ + facilitator: { + lookup: async (host, _question, _timeout, _signal, requestOptions) => { + requestOptions?.consumeBytes?.(1) + return { + type: 'output-list' as const, + outputs: [host === firstHost ? firstOutput : secondOutput] + } + } + }, + hostOverrides: { ls_decoded_budget: [firstHost, secondHost] } + }) + const progress: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$( + { service: 'ls_decoded_budget', query: {} }, + undefined, + { + limits: { + hostConcurrency: 1, + maxTotalBytes: firstOutput.beef.length + firstOutput.context.length + } + } + )) + progress.push(item) + })() + await jest.runAllTimersAsync() + await pending + + expect(progress.at(-1)).toMatchObject({ + isFinal: true, + terminalReason: 'resource-limit', + limitsHit: expect.arrayContaining(['maxTotalBytes']), + receivedBytes: 2, + retainedBytes: firstOutput.beef.length + firstOutput.context.length, + outputs: [firstOutput] + }) + }) + + it('normalizes duplicate configured endpoints and never exceeds the host concurrency budget', async () => { + const one = 'https://one.example' + const two = 'https://two.example' + const three = 'https://three.example' + const calls: string[] = [] + let active = 0 + let peak = 0 + const resolver = new LookupResolver({ + facilitator: { + lookup: async (host: string) => { + calls.push(host) + active++ + peak = Math.max(peak, active) + await later(20) + active-- + return { type: 'output-list' as const, outputs: [] } + } + }, + hostOverrides: { ls_concurrency: [`${one}/`, one, two, three] } + }) + const pending = resolver.query({ service: 'ls_concurrency', query: {} }, undefined, { + limits: { hostConcurrency: 2 } + }) + await jest.runAllTimersAsync() + await pending + + expect(calls).toEqual(expect.arrayContaining([one, two, three])) + expect(calls.filter(host => host === one)).toHaveLength(1) + expect(peak).toBeLessThanOrEqual(2) + }) + + it('gives a later tracker source a turn before draining an earlier tracker flood', async () => { + const firstTracker = 'https://first-tracker.example' + const lateTracker = 'https://late-tracker.example' + const floodHosts = [ + 'https://flood-1.example', + 'https://flood-2.example', + 'https://flood-3.example' + ] + const lateHost = 'https://late-fair.example' + const service = 'ls_fair' + const receipts = await Promise.all([ + ...floodHosts.map((host, index) => slapReceipt(110 + index, host, service)), + slapReceipt(120, lateHost, service) + ]) + const hostCalls: string[] = [] + const resolver = new LookupResolver({ + facilitator: { + lookup: async (host: string) => { + if (host === firstTracker) + return { type: 'output-list' as const, outputs: receipts.slice(0, 3) } + if (host === lateTracker) { + await later(1) + return { type: 'output-list' as const, outputs: [receipts[3]] } + } + hostCalls.push(host) + await later(20) + return { type: 'output-list' as const, outputs: [] } + } + }, + slapTrackers: [firstTracker, lateTracker] + }) + const pending = resolver.query({ service, query: {} }, undefined, { + limits: { hostConcurrency: 1 } + }) + await jest.runAllTimersAsync() + await pending + + expect(hostCalls).toContain(lateHost) + expect(hostCalls.indexOf(lateHost)).toBeLessThan(hostCalls.indexOf(floodHosts[1])) + }) + + it('joins an in-flight refresh even after a stale cached host becomes fresh for a later query', async () => { + const tracker = 'https://refresh-tracker.example' + const cachedHost = 'https://cached-refresh.example' + const lateHost = 'https://late-refresh.example' + const service = 'ls_refresh' + const receipt = await slapReceipt(130, lateHost, service) + const cachedBeef = makeBeef(61) + const lateBeef = makeBeef(62) + let finishRefresh: (() => void) | undefined + const lookup = jest.fn(async (host: string) => { + if (host === tracker) { + await new Promise(resolve => { + finishRefresh = resolve + }) + return { type: 'output-list' as const, outputs: [receipt] } + } + return { + type: 'output-list' as const, + outputs: [{ beef: host === cachedHost ? cachedBeef : lateBeef, outputIndex: 0 }] + } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + ;(resolver as any).hostsCache.set(service, { hosts: [cachedHost], expiresAt: 0 }) + + const first = resolver.query$({ service, query: { caller: 1 } })[Symbol.asyncIterator]() + const firstPending = first.next() + await Promise.resolve() + ;(resolver as any).hostsCache.set(service, { + hosts: [cachedHost], + expiresAt: Date.now() + 60_000 + }) + const secondProgress: LookupAnswerProgress[] = [] + const second = (async () => { + for await (const progress of resolver.query$({ service, query: { caller: 2 } })) { + secondProgress.push(progress) + } + })() + + finishRefresh?.() + await jest.runAllTimersAsync() + await firstPending + await first.return?.() + await second + + expect(lookup.mock.calls.filter(([host]) => host === tracker)).toHaveLength(1) + expect(secondProgress.at(-1)?.outputs).toEqual( + expect.arrayContaining([ + { beef: cachedBeef, outputIndex: 0 }, + { beef: lateBeef, outputIndex: 0 } + ]) + ) + }) + + it('bounds tracker work by trackerConcurrency', async () => { + const trackers = [ + 'https://tracker-one.example', + 'https://tracker-two.example', + 'https://tracker-three.example' + ] + const service = 'ls_tracker_bound' + const host = 'https://tracker-bound-host.example' + const receipt = await slapReceipt(140, host, service) + let active = 0 + let peak = 0 + const resolver = new LookupResolver({ + facilitator: { + lookup: async (url: string) => { + if (trackers.includes(url)) { + active++ + peak = Math.max(peak, active) + await later(20) + active-- + return { type: 'output-list' as const, outputs: url === trackers[0] ? [receipt] : [] } + } + return { type: 'output-list' as const, outputs: [] } + } + }, + slapTrackers: trackers + }) + const pending = resolver.query({ service, query: {} }, undefined, { + limits: { trackerConcurrency: 1 } + }) + await jest.runAllTimersAsync() + await pending + expect(peak).toBe(1) + }) + + it('isolates a slow or throwing evidence listener so the terminal snapshot still arrives', async () => { + const host = 'https://listener.example' + let evidenceCalls = 0 + const resolver = new LookupResolver({ + facilitator: { + lookup: async () => ({ + type: 'output-list' as const, + outputs: [ + { beef: makeBeef(71), outputIndex: 0 }, + { beef: makeBeef(72), outputIndex: 1 } + ] + }) + }, + hostOverrides: { ls_listener: [host] } + }) + const progress: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_listener', query: {} }, undefined, { + onEvidence: () => { + evidenceCalls++ + if (evidenceCalls === 1) return new Promise(() => {}) + throw new Error('consumer failed') + } + })) + progress.push(item) + })() + await jest.runAllTimersAsync() + await pending + expect(progress.at(-1)).toMatchObject({ + isFinal: true, + terminalReason: 'settled', + successfulHosts: 1 + }) + expect(evidenceCalls).toBe(2) + }) + + it('does not reuse a cancelled last-subscriber discovery when its old tracker completes late', async () => { + const tracker = 'https://abandoned-tracker.example' + const staleHost = 'https://abandoned-stale.example' + const freshHost = 'https://abandoned-fresh.example' + const service = 'ls_abandoned' + const staleReceipt = await slapReceipt(150, staleHost, service) + const freshReceipt = await slapReceipt(151, freshHost, service) + let resolveOld: (() => void) | undefined + let trackerCalls = 0 + const lookup = jest.fn(async (url: string) => { + if (url === tracker) { + trackerCalls++ + if (trackerCalls === 1) { + await new Promise(resolve => { + resolveOld = resolve + }) + return { type: 'output-list' as const, outputs: [staleReceipt] } + } + return { type: 'output-list' as const, outputs: [freshReceipt] } + } + return { + type: 'output-list' as const, + outputs: [{ beef: makeBeef(url === freshHost ? 81 : 80), outputIndex: 0 }] + } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + const abort = new AbortController() + const abandoned = resolver + .query$({ service, query: { attempt: 1 } }, undefined, { + signal: abort.signal + }) + [Symbol.asyncIterator]() + const terminal = abandoned.next() + await Promise.resolve() + abort.abort() + await terminal + resolveOld?.() + await Promise.resolve() + + const fresh = resolver.query({ service, query: { attempt: 2 } }) + await jest.runAllTimersAsync() + await expect(fresh).resolves.toEqual({ + type: 'output-list', + outputs: [{ beef: makeBeef(81), outputIndex: 0 }] + }) + expect(trackerCalls).toBe(2) + }) + + it('keeps the ordinary 2s host and 5s tracker attempts within the 10s query deadline', async () => { + const usefulTracker = 'https://useful-4500.example' + const hangingTracker = 'https://hanging-tracker.example' + const discoveredHost = 'https://discovered-1500.example' + const service = 'ls_default_deadline' + const receipt = await slapReceipt(160, discoveredHost, service) + const lookup = jest.fn(async (url: string, _question: unknown, timeout?: number) => { + if (url === usefulTracker) { + expect(timeout).toBe(5000) + await later(4500) + return { type: 'output-list' as const, outputs: [receipt] } + } + if (url === hangingTracker) { + expect(timeout).toBe(5000) + await new Promise(() => {}) + } + expect(timeout).toBeUndefined() + await later(1500) + return { type: 'output-list' as const, outputs: [{ beef: makeBeef(91), outputIndex: 0 }] } + }) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + slapTrackers: [usefulTracker, hangingTracker] + }) + const pending = resolver.query({ service, query: {} }) + await jest.advanceTimersByTimeAsync(6_100) + await expect(pending).resolves.toEqual({ + type: 'output-list', + outputs: [{ beef: makeBeef(91), outputIndex: 0 }] + }) + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.http.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.http.test.ts new file mode 100644 index 000000000..d90721793 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.http.test.ts @@ -0,0 +1,272 @@ +import { createServer } from 'node:http' +import type { Server } from 'node:http' +import type { Socket } from 'node:net' +import LookupResolver, { HTTPSOverlayLookupFacilitator } from '../LookupResolver.js' +import { getOverlayHostReputationTracker } from '../HostReputationTracker.js' +import { Beef, Transaction } from '../../transaction/index.js' +import { LockingScript } from '../../script/index.js' + +interface LookupServer { + url: string + close: () => Promise +} + +/** + * This uses SDK serialization to make a structurally parseable receipt. It is + * deliberately synthetic: parsing BEEF here does not make a cryptographic or + * chain-validity claim. + */ +function structuralOutputListFixture(scriptBytes = 48 * 1024): { + answer: { + type: 'output-list' + outputs: Array<{ beef: number[]; outputIndex: number; context: number[] }> + } + evidenceBytes: number + wire: Buffer +} { + const transaction = new Transaction( + 1, + [], + [{ lockingScript: LockingScript.fromHex('00'.repeat(scriptBytes)), satoshis: 1 }], + 0 + ) + const beef = Beef.fromBinary(transaction.toBEEF()).toBinary() + const answer = { type: 'output-list' as const, outputs: [{ beef, outputIndex: 0 }] } + return { + answer, + evidenceBytes: beef.length, + wire: Buffer.from(JSON.stringify(answer)) + } +} + +function configuredResolver(url: string): LookupResolver { + return new LookupResolver({ + facilitator: new HTTPSOverlayLookupFacilitator(fetch, true), + hostOverrides: { ls_http: [url] } + }) +} + +async function startLookupServer( + handler: Parameters[0] +): Promise { + const server = createServer(handler) + const sockets = new Set() + server.on('connection', socket => { + sockets.add(socket) + socket.once('close', () => sockets.delete(socket)) + }) + + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + const address = server.address() + if (address === null || typeof address === 'string') + throw new Error('Expected a TCP server address') + + return { + url: `http://127.0.0.1:${address.port}`, + close: async () => { + for (const socket of sockets) socket.destroy() + await new Promise((resolve, reject) => { + ;(server as Server).close(error => (error === undefined ? resolve() : reject(error))) + }) + } + } +} + +async function waitForClose(close: Promise): Promise { + let timer: ReturnType | undefined + try { + await Promise.race([ + close, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error('server did not observe client close')), 1500) + }) + ]) + } finally { + if (timer !== undefined) clearTimeout(timer) + } +} + +describe('HTTPSOverlayLookupFacilitator HTTP transport', () => { + let server: LookupServer | undefined + + afterEach(async () => { + await server?.close() + server = undefined + getOverlayHostReputationTracker().reset() + }) + + it('accepts a slowly streamed structural BEEF receipt within a configured budget', async () => { + const fixture = structuralOutputListFixture() + server = await startLookupServer(async (_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }) + for (let offset = 0; offset < fixture.wire.length; offset += 1024) { + response.write(fixture.wire.subarray(offset, offset + 1024)) + await new Promise(resolve => setTimeout(resolve, 2)) + } + response.end() + }) + const resolver = configuredResolver(server.url) + const startedAt = Date.now() + const result = await resolver.queryDetailed({ service: 'ls_http', query: {} }, 2000, { + limits: { maxResponseBytes: fixture.wire.length, maxTotalBytes: fixture.wire.length } + }) + const elapsedMs = Date.now() - startedAt + + expect(result.answer.outputs).toHaveLength(1) + expect(result.answer.outputs[0].beef).toEqual(fixture.answer.outputs[0].beef) + expect(result.progress.receivedBytes).toBe(fixture.wire.length) + expect(fixture.evidenceBytes).toBeGreaterThanOrEqual(48 * 1024) + expect(fixture.wire.length).toBeGreaterThan(fixture.evidenceBytes) + expect({ evidenceBytes: fixture.evidenceBytes, jsonWireBytes: fixture.wire.length }).toEqual({ + evidenceBytes: 49_180, + jsonWireBytes: 98_429 + }) + expect(elapsedMs).toBeGreaterThanOrEqual(20) + expect(elapsedMs).toBeLessThan(2000) + }) + + it('limits evidence callbacks for the structural receipt, then admits it through evidenceLimits', async () => { + const fixture = structuralOutputListFixture() + server = await startLookupServer((_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(fixture.wire) + }) + const limitedEvidence: string[] = [] + const limited = await configuredResolver(server.url).queryDetailed( + { service: 'ls_http', query: {} }, + 1000, + { + limits: { + maxResponseBytes: fixture.wire.length, + maxTotalBytes: fixture.wire.length, + maxEvidenceBytes: fixture.evidenceBytes - 1 + }, + onEvidence: event => limitedEvidence.push(event.type) + } + ) + const admittedEvidence: string[] = [] + const admitted = await configuredResolver(server.url).queryDetailed( + { service: 'ls_http', query: {} }, + 1000, + { + limits: { maxResponseBytes: fixture.wire.length, maxTotalBytes: fixture.wire.length }, + evidenceLimits: { maxBytes: fixture.evidenceBytes }, + onEvidence: event => admittedEvidence.push(event.type) + } + ) + + expect(limitedEvidence).toEqual(['limit']) + expect(limited.progress.terminalReason).toBe('resource-limit') + expect(limited.progress.limitsHit).toContain('maxEvidenceBytes') + expect(admittedEvidence).toEqual(['output']) + expect(admitted.progress.terminalReason).toBe('settled') + expect(admitted.progress.receivedBytes).toBe(fixture.wire.length) + }) + + it.each(['maxResponseBytes', 'maxTotalBytes'] as const)( + 'reports %s exhaustion without recording an availability failure', + async limitName => { + const fixture = structuralOutputListFixture() + server = await startLookupServer((_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(fixture.wire) + }) + const smallLimit = fixture.wire.length - 1 + const result = await configuredResolver(server.url).queryDetailed( + { service: 'ls_http', query: {} }, + 1000, + { + limits: + limitName === 'maxResponseBytes' + ? { maxResponseBytes: smallLimit, maxTotalBytes: fixture.wire.length } + : { maxResponseBytes: fixture.wire.length, maxTotalBytes: smallLimit } + } + ) + + expect(result.answer.outputs).toEqual([]) + expect(result.progress.terminalReason).toBe('resource-limit') + expect(result.progress.limitsHit).toContain(limitName) + expect(result.progress.completedHosts).toBe(1) + expect(getOverlayHostReputationTracker().snapshot(server.url)).toMatchObject({ + totalFailures: 0, + consecutiveFailures: 0, + totalSuccesses: 0 + }) + } + ) + + it('times out a response whose body never finishes and closes the server-side response', async () => { + let resolveClosed: () => void = () => undefined + const responseClosed = new Promise(resolve => { + resolveClosed = resolve + }) + server = await startLookupServer((_request, response) => { + response.once('close', resolveClosed) + response.writeHead(200, { 'content-type': 'application/json' }) + response.write('{"type":"freeform","result":"') + }) + const facilitator = new HTTPSOverlayLookupFacilitator(fetch, true) + + await expect( + facilitator.lookup(server.url, { service: 'ls_http', query: {} }, 50, undefined, { + maxResponseBytes: 1024, + maxOutputs: 1 + }) + ).rejects.toThrow('Request timed out') + + await waitForClose(responseClosed) + }) + + it.each([ + ['invalid UTF-8', Buffer.from([0xff, 0xfe])], + ['malformed JSON', Buffer.from('{')] + ])('rejects %s without treating it as a successful answer', async (_name, payload) => { + server = await startLookupServer((_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(payload) + }) + const facilitator = new HTTPSOverlayLookupFacilitator(fetch, true) + + await expect( + facilitator.lookup(server.url, { service: 'ls_http', query: {} }, 1000, undefined, { + maxResponseBytes: 1024, + maxOutputs: 1 + }) + ).rejects.toBeInstanceOf(Error) + }) + + it('honors an early caller abort and closes the server-side response', async () => { + let resolveStarted: () => void = () => undefined + const started = new Promise(resolve => { + resolveStarted = resolve + }) + let resolveClosed: () => void = () => undefined + const responseClosed = new Promise(resolve => { + resolveClosed = resolve + }) + server = await startLookupServer((_request, response) => { + response.once('close', resolveClosed) + response.writeHead(200, { 'content-type': 'application/json' }) + response.write('{"type":"freeform","result":"') + resolveStarted() + }) + const facilitator = new HTTPSOverlayLookupFacilitator(fetch, true) + const controller = new AbortController() + const lookup = facilitator.lookup( + server.url, + { service: 'ls_http', query: {} }, + 2000, + controller.signal, + { maxResponseBytes: 1024, maxOutputs: 1 } + ) + + await started + controller.abort() + + await expect(lookup).rejects.toMatchObject({ name: 'AbortError', message: 'Lookup cancelled' }) + await waitForClose(responseClosed) + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.test.ts index b0072979e..88cd1f916 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.test.ts @@ -10,6 +10,23 @@ const mockFacilitator = { lookup: jest.fn() } +const expectLookupCalls = (actual: unknown[][], expected: unknown[][]): void => { + expect(actual.map(call => call.slice(0, 3))).toEqual(expected) + for (const call of actual) { + expect(call[3]).toEqual(expect.any(AbortSignal)) + expect(call[4]).toEqual( + expect.objectContaining({ + maxResponseBytes: 32 * 1024 * 1024, + consumeBytes: expect.any(Function) + }) + ) + } +} + +const expectLookupCall = (actual: unknown[], expected: unknown[]): void => { + expectLookupCalls([actual], [expected]) +} + const sampleBeef1 = new Transaction( 1, [], @@ -96,7 +113,7 @@ describe('LookupResolver', () => { } ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -137,40 +154,24 @@ describe('LookupResolver', () => { 0 ) - mockFacilitator.lookup - .mockReturnValueOnce({ - type: 'output-list', - outputs: [ - { - outputIndex: 0, - beef: slapTx.toBEEF() - } - ] - }) - .mockReturnValueOnce({ - type: 'output-list', - outputs: [ - { - beef: sampleBeef1, - outputIndex: 0 - } - ] - }) - .mockReturnValueOnce({ + mockFacilitator.lookup.mockImplementation((url: string, question: { service: string }) => { + if (question.service === 'ls_slap') { + return { + type: 'output-list', + outputs: [{ outputIndex: 0, beef: slapTx.toBEEF() }] + } + } + if (url === 'https://slaphost.com') { + return { type: 'output-list', outputs: [{ beef: sampleBeef1, outputIndex: 0 }] } + } + return { type: 'output-list', outputs: [ - { - // duplicate the output the other host knows about - beef: sampleBeef1, - outputIndex: 0 - }, - { - // the additional host also knows about a second output - beef: sampleBeef2, - outputIndex: 1033 - } + { beef: sampleBeef1, outputIndex: 0 }, + { beef: sampleBeef2, outputIndex: 1033 } ] - }) + } + }) const r = new LookupResolver({ facilitator: mockFacilitator, @@ -198,30 +199,30 @@ describe('LookupResolver', () => { } ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ - 'https://mock.slap', + // additional host should also have been queried first + 'https://additional.host', { - service: 'ls_slap', + service: 'ls_foo', query: { - service: 'ls_foo' + test: 1 } }, - 5000 + undefined ], [ - 'https://slaphost.com', + 'https://mock.slap', { - service: 'ls_foo', + service: 'ls_slap', query: { - test: 1 + service: 'ls_foo' } }, - undefined + 5000 ], [ - // additional host should also have been queried - 'https://additional.host', + 'https://slaphost.com', { service: 'ls_foo', query: { @@ -264,7 +265,7 @@ describe('LookupResolver', () => { } ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://override.host', { @@ -332,7 +333,7 @@ describe('LookupResolver', () => { } ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://override.host', { @@ -357,7 +358,7 @@ describe('LookupResolver', () => { ]) }) - it('should handle multiple SLAP trackers and resolve with first responder hosts', async () => { + it('queries every eligible host advertised during the active attempt, including a later tracker', async () => { const slapHostKey1 = new PrivateKey(42) const slapWallet1 = new CompletedProtoWallet(slapHostKey1) const slapLib1 = new OverlayAdminTokenTemplate(slapWallet1) @@ -390,36 +391,28 @@ describe('LookupResolver', () => { 0 ) - // SLAP trackers return hosts — first responder wins - mockFacilitator.lookup - .mockReturnValueOnce({ - type: 'output-list', - outputs: [ - { - outputIndex: 0, - beef: slapTx1.toBEEF() + mockFacilitator.lookup.mockImplementation((url: string, question: { service: string }) => { + if (question.service === 'ls_slap') { + if (url === 'https://mock.slap1') { + return { + type: 'output-list', + outputs: [{ outputIndex: 0, beef: slapTx1.toBEEF() }] } - ] - }) - .mockReturnValueOnce({ - type: 'output-list', - outputs: [ - { - outputIndex: 0, - beef: slapTx2.toBEEF() + } + if (url === 'https://mock.slap2') { + return { + type: 'output-list', + outputs: [{ outputIndex: 0, beef: slapTx2.toBEEF() }] } - ] - }) - - // Only the first-resolved tracker's host gets queried - mockFacilitator.lookup.mockReturnValueOnce({ - type: 'output-list', - outputs: [ - { - beef: sampleBeef3, - outputIndex: 0 } - ] + } + if (url === 'https://slaphost1.com') { + return { type: 'output-list', outputs: [{ beef: sampleBeef3, outputIndex: 0 }] } + } + if (url === 'https://slaphost2.com') { + return { type: 'output-list', outputs: [{ beef: sampleBeef2, outputIndex: 1 }] } + } + throw new Error(`unexpected host ${url}`) }) const r = new LookupResolver({ @@ -432,15 +425,24 @@ describe('LookupResolver', () => { query: { test: 1 } }) - // Only the first tracker's host results are returned - expect(res).toEqual({ - type: 'output-list', - outputs: [{ beef: sampleBeef3, outputIndex: 0 }] - }) + expect(res.outputs).toEqual( + expect.arrayContaining([ + { beef: sampleBeef3, outputIndex: 0 }, + { beef: sampleBeef2, outputIndex: 1 } + ]) + ) + expect(res.outputs).toHaveLength(2) - // Both SLAP trackers are queried, but only the first host is used for the actual query - expect(mockFacilitator.lookup.mock.calls.length).toBeGreaterThanOrEqual(3) - expect(mockFacilitator.lookup.mock.calls[0]).toEqual([ + const calledUrls = mockFacilitator.lookup.mock.calls.map((call: unknown[]) => call[0]) + expect(calledUrls).toEqual( + expect.arrayContaining([ + 'https://mock.slap1', + 'https://mock.slap2', + 'https://slaphost1.com', + 'https://slaphost2.com' + ]) + ) + expectLookupCall(mockFacilitator.lookup.mock.calls[0], [ 'https://mock.slap1', { service: 'ls_slap', @@ -450,7 +452,7 @@ describe('LookupResolver', () => { }, 5000 ]) - expect(mockFacilitator.lookup.mock.calls[1]).toEqual([ + expectLookupCall(mockFacilitator.lookup.mock.calls[1], [ 'https://mock.slap2', { service: 'ls_slap', @@ -460,16 +462,6 @@ describe('LookupResolver', () => { }, 5000 ]) - expect(mockFacilitator.lookup.mock.calls[2]).toEqual([ - 'https://slaphost1.com', - { - service: 'ls_foo', - query: { - test: 1 - } - }, - undefined - ]) }) it('should de-duplicate outputs from multiple hosts', async () => { @@ -541,7 +533,7 @@ describe('LookupResolver', () => { outputs: [duplicateOutput] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -644,7 +636,7 @@ describe('LookupResolver', () => { outputs: [{ beef: sampleBeef3, outputIndex: 0 }] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -699,7 +691,7 @@ describe('LookupResolver', () => { 'No competent mainnet hosts found by the SLAP trackers for lookup service: ls_foo' ) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -786,7 +778,7 @@ describe('LookupResolver', () => { ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -848,7 +840,7 @@ describe('LookupResolver', () => { } ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -884,7 +876,7 @@ describe('LookupResolver', () => { 'No competent mainnet hosts found by the SLAP trackers for lookup service: ls_foo' ) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -1115,7 +1107,7 @@ describe('LookupResolver', () => { ] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap1', { @@ -1238,7 +1230,7 @@ describe('LookupResolver', () => { outputs: [{ beef: sampleBeef3, outputIndex: 0 }] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -1341,7 +1333,7 @@ describe('LookupResolver', () => { outputs: [{ beef: sampleBeef3, outputIndex: 0 }] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -1443,7 +1435,7 @@ describe('LookupResolver', () => { outputs: [{ beef: sampleBeef3, outputIndex: 0 }] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { @@ -1522,7 +1514,7 @@ describe('LookupResolver', () => { outputs: [] }) - expect(mockFacilitator.lookup.mock.calls).toEqual([ + expectLookupCalls(mockFacilitator.lookup.mock.calls, [ [ 'https://mock.slap', { diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResponseReader.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResponseReader.test.ts new file mode 100644 index 000000000..2cdf8d075 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResponseReader.test.ts @@ -0,0 +1,243 @@ +import { readLookupResponseBytes } from '../LookupResponseReader.js' +import { LookupResourceLimitError } from '../LookupResources.js' + +function responseForReader( + reader: ReadableStreamDefaultReader, + contentLength?: string +): Response { + const headers = new Headers() + if (contentLength !== undefined) headers.set('content-length', contentLength) + return { + body: { getReader: () => reader }, + headers + } as unknown as Response +} + +function readerForChunks(chunks: Uint8Array[]): ReadableStreamDefaultReader { + let index = 0 + return { + read: async () => + index < chunks.length + ? { done: false, value: chunks[index++] } + : { done: true, value: undefined }, + cancel: async () => undefined, + releaseLock: () => undefined + } as unknown as ReadableStreamDefaultReader +} + +describe('readLookupResponseBytes', () => { + it('joins normal streamed chunks and charges each accepted chunk', async () => { + const consumed: number[] = [] + const bytes = await readLookupResponseBytes( + responseForReader(readerForChunks([new Uint8Array([1, 2]), new Uint8Array([3, 4, 5])])), + { maxResponseBytes: 5, consumeBytes: byteCount => consumed.push(byteCount) } + ) + + expect(bytes).toEqual(new Uint8Array([1, 2, 3, 4, 5])) + expect(consumed).toEqual([2, 3]) + }) + + it('allows a body exactly at the configured boundary', async () => { + const bytes = await readLookupResponseBytes( + responseForReader(readerForChunks([new Uint8Array([1, 2]), new Uint8Array([3])]), '3'), + { maxResponseBytes: 3 } + ) + + expect(bytes).toEqual(new Uint8Array([1, 2, 3])) + }) + + it('rejects a known oversized Content-Length before reading chunks', async () => { + const read = jest.fn() + const cancel = jest.fn(async () => undefined) + const releaseLock = jest.fn() + const reader = { + read, + cancel, + releaseLock + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader, '6'), { maxResponseBytes: 5 }) + ).rejects.toMatchObject({ name: 'LookupResourceLimitError', limit: 'maxResponseBytes' }) + + expect(read).not.toHaveBeenCalled() + expect(cancel).toHaveBeenCalledTimes(1) + expect(releaseLock).toHaveBeenCalledTimes(1) + }) + + it('rejects an overflowing chunk before it is charged or accumulated', async () => { + const cancel = jest.fn(async () => undefined) + const releaseLock = jest.fn() + const reader = { + read: jest + .fn() + .mockResolvedValueOnce({ done: false, value: new Uint8Array([1, 2, 3]) }) + .mockResolvedValueOnce({ done: false, value: new Uint8Array([4, 5, 6]) }), + cancel, + releaseLock + } as unknown as ReadableStreamDefaultReader + const consumed: number[] = [] + + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 5, + consumeBytes: byteCount => consumed.push(byteCount) + }) + ).rejects.toBeInstanceOf(LookupResourceLimitError) + + expect(consumed).toEqual([3]) + expect(cancel).toHaveBeenCalledTimes(1) + expect(releaseLock).toHaveBeenCalledTimes(1) + }) + + it('copies producer-owned chunk buffers before the producer reuses them', async () => { + const producerBuffer = new Uint8Array([1, 2]) + let readCount = 0 + const reader = { + read: async () => { + readCount += 1 + if (readCount === 1) return { done: false, value: producerBuffer } + if (readCount === 2) { + producerBuffer.set([3, 4]) + return { done: false, value: producerBuffer } + } + return { done: true, value: undefined } + }, + cancel: async () => undefined, + releaseLock: () => undefined + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader), { maxResponseBytes: 4 }) + ).resolves.toEqual(new Uint8Array([1, 2, 3, 4])) + }) + + it('ignores many empty chunks while retaining tiny chunks in bounded storage', async () => { + const emptyChunks = Array.from({ length: 2048 }, () => new Uint8Array(0)) + const tinyChunks = Array.from({ length: 32 }, (_unused, index) => new Uint8Array([index])) + const consumed: number[] = [] + + const bytes = await readLookupResponseBytes( + responseForReader(readerForChunks([...emptyChunks, ...tinyChunks])), + { maxResponseBytes: 32, consumeBytes: byteCount => consumed.push(byteCount) } + ) + + expect(bytes).toEqual(new Uint8Array(Array.from({ length: 32 }, (_unused, index) => index))) + expect(consumed).toEqual(Array.from({ length: 32 }, () => 1)) + }) + + it('yields so a timer abort can stop an endless eager empty stream', async () => { + const controller = new AbortController() + const cancel = jest.fn(async () => undefined) + const reader = { + read: jest.fn(async () => ({ done: false, value: new Uint8Array(0) })), + cancel, + releaseLock: jest.fn() + } as unknown as ReadableStreamDefaultReader + const timer = setTimeout(() => controller.abort(new Error('empty stream aborted')), 1) + + try { + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 1, + signal: controller.signal + }) + ).rejects.toThrow('empty stream aborted') + } finally { + clearTimeout(timer) + } + + expect(cancel).toHaveBeenCalledTimes(1) + expect((reader.read as jest.Mock).mock.calls.length).toBeGreaterThanOrEqual(64) + expect((reader.read as jest.Mock).mock.calls.length).toBeLessThan(256) + }) + + it('copies the sixty-fourth chunk before yielding to timer-driven producer mutation', async () => { + const producerBuffer = new Uint8Array([1]) + let reads = 0 + const reader = { + read: async () => { + reads += 1 + if (reads <= 64) { + if (reads === 64) setTimeout(() => producerBuffer.fill(9), 0) + return { done: false, value: producerBuffer } + } + return { done: true, value: undefined } + }, + cancel: async () => undefined, + releaseLock: () => undefined + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader), { maxResponseBytes: 64 }) + ).resolves.toEqual(new Uint8Array(64).fill(1)) + }) + + it('cleans up when the aggregate byte budget rejects an accepted chunk', async () => { + const cancel = jest.fn(async () => undefined) + const releaseLock = jest.fn() + const reader = { + read: jest.fn().mockResolvedValue({ done: false, value: new Uint8Array([1, 2]) }), + cancel, + releaseLock + } as unknown as ReadableStreamDefaultReader + const budgetFailure = new Error('aggregate response budget exhausted') + + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 10, + consumeBytes: () => { + throw budgetFailure + } + }) + ).rejects.toBe(budgetFailure) + + expect(cancel).toHaveBeenCalledWith(budgetFailure) + expect(releaseLock).toHaveBeenCalledTimes(1) + }) + + it('rejects an abort while a reader read remains pending and starts cleanup', async () => { + let rejectRead: (reason?: unknown) => void = () => undefined + const pendingRead = new Promise>((_, reject) => { + rejectRead = reject + }) + const cancel = jest.fn(() => { + rejectRead(new Error('cancelled pending read')) + return Promise.resolve() + }) + const releaseLock = jest.fn() + const reader = { + read: jest.fn(() => pendingRead), + cancel, + releaseLock + } as unknown as ReadableStreamDefaultReader + const controller = new AbortController() + const aborted = readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 10, + signal: controller.signal + }) + + controller.abort(new Error('lookup aborted')) + + await expect(aborted).rejects.toThrow('lookup aborted') + expect(cancel).toHaveBeenCalledTimes(1) + expect(releaseLock).toHaveBeenCalledTimes(1) + }) + + it('preserves the read failure when cancellation and lock release throw', async () => { + const readFailure = new Error('stream failed') + const reader = { + read: jest.fn().mockRejectedValue(readFailure), + cancel: jest.fn(() => { + throw new Error('cancel failed') + }), + releaseLock: jest.fn(() => { + throw new Error('release failed') + }) + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader), { maxResponseBytes: 10 }) + ).rejects.toBe(readFailure) + }) +}) From 0e5586b856b4f875a24072ab2b79bcf5827be8d4 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 14:53:44 -0500 Subject: [PATCH 006/168] feat(overlay): add opt-in Mongo payload and schema foundation Add replica-set schema bootstrap, content-addressed payload publication, transactionally guarded references/GC, and majority-commit outcome helpers behind an optional mongodb peer. This is not an AdmissionStorage adapter and does not change Engine or Knex defaults. --- docs/packages/overlays/overlay.md | 19 + docs/reference/package-api-migrations.md | 6 +- governance/package-release-notes.json | 4 +- packages/overlays/overlay/CHANGELOG.md | 4 + packages/overlays/overlay/README.md | 28 +- packages/overlays/overlay/package.json | 10 +- .../mongo/MongoCommitResponseLoss.test.ts | 144 +++ .../__tests/mongo/MongoCommitResponseProxy.ts | 212 ++++ .../__tests/mongo/MongoPayloadStore.test.ts | 564 ++++++++++ .../src/__tests/mongo/MongoReadGuards.test.ts | 113 ++ .../src/__tests/mongo/MongoReplicaFixture.ts | 146 +++ .../src/__tests/mongo/MongoSchema.test.ts | 221 ++++ .../mongo/MongoTransactionRunner.test.ts | 314 ++++++ .../overlays/overlay/src/storage/mongo.ts | 5 + .../src/storage/mongo/MongoPayloadStore.ts | 889 ++++++++++++++++ .../src/storage/mongo/MongoReadGuards.ts | 209 ++++ .../overlay/src/storage/mongo/MongoSchema.ts | 964 ++++++++++++++++++ .../storage/mongo/MongoTransactionRunner.ts | 376 +++++++ pnpm-lock.yaml | 6 + specs/overlay/mongo-v1.md | 123 +++ 20 files changed, 4348 insertions(+), 9 deletions(-) create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseLoss.test.ts create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseProxy.ts create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoReadGuards.test.ts create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoReplicaFixture.ts create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts create mode 100644 packages/overlays/overlay/src/storage/mongo.ts create mode 100644 packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts create mode 100644 packages/overlays/overlay/src/storage/mongo/MongoReadGuards.ts create mode 100644 packages/overlays/overlay/src/storage/mongo/MongoSchema.ts create mode 100644 packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts create mode 100644 specs/overlay/mongo-v1.md diff --git a/docs/packages/overlays/overlay.md b/docs/packages/overlays/overlay.md index 8e7d462ed..38a99b04a 100644 --- a/docs/packages/overlays/overlay.md +++ b/docs/packages/overlays/overlay.md @@ -85,6 +85,25 @@ receipt and pending index/propagation boundary for future adapters. Current capability. See the [persistence v1 specification](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/persistence-v1.md) for the shared fixtures and explicit limits. No consumer migration is required. +## Optional Mongo foundation + +The package also contains an opt-in MongoDB foundation for schema bootstrap, +content-addressed payload publication, reference guards, and payload collection. +It is not an Engine integration, an `AdmissionStorage` implementation, or a +default storage selection; importing `@bsv/overlay` does not load MongoDB. + +Applications using a Mongo deep entry point install the optional peer first: + +```sh +npm install @bsv/overlay mongodb@^7.5.0 +``` + +The initial entry points are `@bsv/overlay/storage/mongo/MongoSchema` and +`@bsv/overlay/storage/mongo/MongoPayloadStore`. They require an explicitly +operated unsharded replica set; the supported deployment profile is three +members. See the [Mongo v1 +foundation](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/mongo-v1.md). + ## Common patterns ### Implementing a TopicManager diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 1aea5d534..b861c5c49 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -42,7 +42,7 @@ and clean-consumer tests remain the executable type authority. | `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | | `@bsv/message-box-client` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | -| `@bsv/overlay` | `2.2.1` | `2.4.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet, and no Mongo/runtime activation or mixed-writer database topology is included. | +| `@bsv/overlay` | `2.2.1` | `2.4.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. | | `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.2` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | | `@bsv/overlay-express` | `2.5.0` | `2.6.2` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | | `@bsv/overlay-topics` | `1.6.10` | `1.7.2` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | @@ -277,8 +277,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay.md](../packages/overlays/overlay.md) - Source: [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) -- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. -- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet, and no Mongo/runtime activation or mixed-writer database topology is included. +- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. +- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index 664160c78..2720237f9 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -126,8 +126,8 @@ "name": "@bsv/overlay", "publishedVersion": "2.2.1", "releaseType": "minor", - "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures.", - "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet, and no Mongo/runtime activation or mixed-writer database topology is included." + "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection.", + "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included." }, { "name": "@bsv/overlay-discovery-services", diff --git a/packages/overlays/overlay/CHANGELOG.md b/packages/overlays/overlay/CHANGELOG.md index e90ebaad1..64c003ec8 100644 --- a/packages/overlays/overlay/CHANGELOG.md +++ b/packages/overlays/overlay/CHANGELOG.md @@ -21,6 +21,10 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] - Adds optional atomic admission/receipt and recovery contracts with portable identity, exact integer and fencing fixtures. Existing Engine/Knex paths are unchanged; no migration is required. +- Adds an opt-in MongoDB schema, payload-publication, reference-guard, and + payload-GC foundation behind an optional `mongodb` peer. It does not activate + MongoDB in Engine, select a new default adapter, or provide an admission + receipt path. ### Added - (Include new features or significant user-visible enhancements here.) diff --git a/packages/overlays/overlay/README.md b/packages/overlays/overlay/README.md index 1525ba066..891b16f70 100644 --- a/packages/overlays/overlay/README.md +++ b/packages/overlays/overlay/README.md @@ -94,8 +94,32 @@ uses ready payload references and outbox intents; and fences recovery by both chain epoch and topic history generation. Helper functions and shared fixtures pin exact integers, deterministic identity, leases and cursor eligibility. See the [persistence specification](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/persistence-v1.md). -No consumer migration, Mongo runtime, storage-default change or database migration -is included in this release candidate. +No consumer migration, storage-default change or database migration is included +in this release candidate. + +## Optional Mongo foundation + +The package also contains an opt-in MongoDB foundation for schema bootstrap, +content-addressed payload publication, reference guards, and payload collection. +It is not an Engine integration, an `AdmissionStorage` implementation, or a +default storage selection; importing `@bsv/overlay` alone does not load MongoDB. + +Applications using a Mongo deep entry point install the optional peer first: + +```sh +npm install @bsv/overlay mongodb@^7.5.0 +``` + +The initial entry points are +`@bsv/overlay/storage/mongo/MongoSchema` and +`@bsv/overlay/storage/mongo/MongoPayloadStore`. They require an explicitly +operated unsharded replica set; the supported deployment profile is three +members. Payload publication makes GridFS bytes physically `published` before +the guarded payload row becomes `ready`; caller-session reference and GC +operations share that row guard. See the [Mongo v1 +foundation](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/mongo-v1.md) +for operational bounds, recovery rules, and the still-pending admission +transaction integration. ## Runtime and package formats diff --git a/packages/overlays/overlay/package.json b/packages/overlays/overlay/package.json index 1b5a200c9..cdad98132 100644 --- a/packages/overlays/overlay/package.json +++ b/packages/overlays/overlay/package.json @@ -117,18 +117,24 @@ "ts-jest": "^29.4.12", "ts2md": "^0.2.8", "tsconfig-to-dual-package": "^1.2.0", - "typescript": "npm:@typescript/typescript6@6.0.2" + "typescript": "npm:@typescript/typescript6@6.0.2", + "mongodb": "7.5.0", + "mongodb-memory-server": "^11.2.0" }, "dependencies": { "@bsv/gasp": "workspace:^", "knex": "^3.3.0" }, "peerDependencies": { - "@bsv/sdk": "^2.1.6" + "@bsv/sdk": "^2.1.6", + "mongodb": "^7.5.0" }, "peerDependenciesMeta": { "@bsv/sdk": { "optional": false + }, + "mongodb": { + "optional": true } }, "typesVersions": { diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseLoss.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseLoss.test.ts new file mode 100644 index 000000000..85dbb9e64 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseLoss.test.ts @@ -0,0 +1,144 @@ +import { randomUUID } from 'node:crypto' +import { MongoClient } from 'mongodb' +import { admissionSemanticDigest, type AdmissionIdentity } from '../../storage/AdmissionStorage.js' +import { + MongoTransactionRunner, + type MongoTransactionRequest +} from '../../storage/mongo/MongoTransactionRunner.js' +import { bootstrapMongoOverlay } from '../../storage/mongo/MongoSchema.js' +import { MongoCommitResponseProxy } from './MongoCommitResponseProxy.js' +import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' + +describe('Mongo commit response loss', () => { + let fixture: MongoReplicaFixture + + function request(operationId = randomUUID()): MongoTransactionRequest { + const identity: AdmissionIdentity = { + scope: fixture.scope, + txid: '52'.repeat(32), + mode: 'live', + contextDigest: '63'.repeat(32), + topics: [{ topic: 'commit.loss', policyId: 'v1' }] + } + const semanticDigest = admissionSemanticDigest(identity) + return { + identity, + key: { scope: fixture.scope, operationId, semanticDigest }, + receipt: { + operationId, + semanticDigest, + durability: 'atomic-local', + steak: '{"commit.loss":{"outputsToAdmit":[0]}}', + indexes: [{ target: 'lookup', state: 'pending' }], + propagation: 'pending' + } + } + } + + async function lostReplyClient(): Promise<{ + client: MongoClient + proxy: MongoCommitResponseProxy + }> { + const hello = await fixture.db.admin().command({ hello: 1 }) + if (typeof hello.primary !== 'string') throw new Error('Replica fixture has no primary') + const proxy = await MongoCommitResponseProxy.create(hello.primary) + const client = new MongoClient(proxy.uri, { + directConnection: true, + maxAdaptiveRetries: 0, + retryReads: false, + retryWrites: false, + serverSelectionTimeoutMS: 15000, + connectTimeoutMS: 5000, + maxPoolSize: 2, + writeConcern: { w: 'majority', j: true } + }) + await client.connect() + return { client, proxy } + } + + beforeAll(async () => { + fixture = await createMongoReplicaFixture() + await bootstrapMongoOverlay(fixture.db, fixture.scope) + }, 90000) + + afterAll(async () => { + await fixture.close() + }) + + test('reconciles a successful commit whose real server response was dropped using the same session', async () => { + const { client, proxy } = await lostReplyClient() + const runner = new MongoTransactionRunner(client.db(fixture.db.databaseName), fixture.scope, { + maxCommitAttempts: 1 + }) + const input = request() + let bodies = 0 + try { + const pending = await runner.run( + input, + async context => { + bodies += 1 + await client + .db(fixture.db.databaseName) + .collection('response_loss_effects') + .insertOne({ operationId: input.key.operationId }, context.options()) + }, + { timeoutMS: 15000 } + ) + await proxy.waitForDroppedReply() + expect(proxy.droppedSuccessfulCommitReply).toBe(true) + expect(pending.state).toBe('pending') + const attemptId = pending.state === 'pending' ? pending.attemptId : undefined + expect(await runner.reconcile(input.key, attemptId, { timeoutMS: 15000 })).toEqual({ + state: 'committed', + receipt: input.receipt + }) + expect(bodies).toBe(1) + expect(proxy.commits).toHaveLength(1) + expect(proxy.commits[0].lsid).not.toHaveLength(0) + expect(proxy.commits[0].txnNumber).toBe('1') + } finally { + await runner.close() + await client.close() + await proxy.close() + } + }, 30000) + + test('replays the majority receipt after disposing the runner that lost the response', async () => { + const { client, proxy } = await lostReplyClient() + const runner = new MongoTransactionRunner(client.db(fixture.db.databaseName), fixture.scope, { + maxCommitAttempts: 1 + }) + const input = request() + let bodies = 0 + try { + const pending = await runner.run( + input, + async () => { + bodies += 1 + }, + { timeoutMS: 15000 } + ) + await proxy.waitForDroppedReply() + expect(proxy.droppedSuccessfulCommitReply).toBe(true) + expect(pending.state).toBe('pending') + await runner.close() + const restarted = new MongoTransactionRunner(fixture.db, fixture.scope) + try { + expect( + await restarted.run(input, async () => { + bodies += 1 + }) + ).toEqual({ + state: 'committed', + receipt: input.receipt + }) + } finally { + await restarted.close() + } + expect(bodies).toBe(1) + } finally { + await client.close() + await proxy.close() + } + }, 30000) +}) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseProxy.ts b/packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseProxy.ts new file mode 100644 index 000000000..0e0e62bc1 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseProxy.ts @@ -0,0 +1,212 @@ +import net from 'node:net' +import { BSON } from 'mongodb' + +const maxFrameBytes = 4 * 1024 * 1024 +const maxBufferedBytes = maxFrameBytes * 2 +const opMessage = 2013 + +export interface CapturedMongoCommit { + readonly requestId: number + readonly lsid: string + readonly txnNumber: string +} + +interface Connection { + client: net.Socket + upstream: net.Socket + clientBuffer: Buffer + upstreamBuffer: Buffer + commitRequests: Set +} + +/** + * Test-only bounded TCP relay that drops exactly one successful OP_MSG + * commitTransaction reply after MongoDB has sent it to the relay. + */ +export class MongoCommitResponseProxy { + public readonly uri: string + public readonly commits: CapturedMongoCommit[] = [] + public droppedSuccessfulCommitReply = false + private readonly connections = new Set() + private dropped = false + private readonly droppedReply: Promise + private resolveDropped!: () => void + + private constructor( + private readonly server: net.Server, + private readonly targetHost: string, + private readonly targetPort: number, + port: number + ) { + this.uri = `mongodb://127.0.0.1:${port}` + this.droppedReply = new Promise(resolve => { + this.resolveDropped = resolve + }) + } + + public static async create(target: string): Promise { + const url = new URL(`mongodb://${target}`) + const host = url.hostname + const port = Number(url.port) + if (host.length === 0 || !Number.isSafeInteger(port) || port < 1 || port > 65535) + throw new Error('Invalid Mongo proxy target') + let proxy: MongoCommitResponseProxy | undefined + const server = net.createServer(client => proxy?.accept(client)) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + server.off('error', reject) + resolve() + }) + }) + const address = server.address() + if (address === null || typeof address === 'string') { + await new Promise(resolve => server.close(() => resolve())) + throw new Error('Mongo proxy did not bind a TCP port') + } + proxy = new MongoCommitResponseProxy(server, host, port, address.port) + return proxy + } + + public async waitForDroppedReply(timeoutMS = 15000): Promise { + if (!Number.isSafeInteger(timeoutMS) || timeoutMS < 1 || timeoutMS > 30000) + throw new Error('Invalid Mongo proxy timeout') + await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error('Mongo proxy did not observe a commit reply')), + timeoutMS + ) + timer.unref() + this.droppedReply.then( + () => { + clearTimeout(timer) + resolve() + }, + error => { + clearTimeout(timer) + reject(error) + } + ) + }) + } + + public async close(): Promise { + for (const connection of this.connections) { + connection.client.destroy() + connection.upstream.destroy() + } + await new Promise(resolve => this.server.close(() => resolve())) + } + + private accept(client: net.Socket): void { + const upstream = net.createConnection({ host: this.targetHost, port: this.targetPort }) + const connection: Connection = { + client, + upstream, + clientBuffer: Buffer.alloc(0), + upstreamBuffer: Buffer.alloc(0), + commitRequests: new Set() + } + this.connections.add(connection) + let closed = false + const close = () => { + if (closed) return + closed = true + this.connections.delete(connection) + client.destroy() + upstream.destroy() + } + client.once('close', close) + upstream.once('close', close) + client.on('error', close) + upstream.on('error', close) + client.on('data', data => this.forward(connection, 'client', data)) + upstream.on('data', data => this.forward(connection, 'upstream', data)) + } + + private forward(connection: Connection, direction: 'client' | 'upstream', data: Buffer): void { + const buffer = Buffer.concat([ + direction === 'client' ? connection.clientBuffer : connection.upstreamBuffer, + data + ]) + if (buffer.byteLength > maxBufferedBytes) { + connection.client.destroy(new Error('Mongo proxy frame buffer exceeded')) + connection.upstream.destroy() + return + } + let offset = 0 + while (buffer.byteLength - offset >= 16) { + const length = buffer.readInt32LE(offset) + if (length < 16 || length > maxFrameBytes) { + connection.client.destroy(new Error('Mongo proxy received invalid frame length')) + connection.upstream.destroy() + return + } + if (buffer.byteLength - offset < length) break + const message = buffer.subarray(offset, offset + length) + offset += length + if (direction === 'client') { + this.observeRequest(connection, message) + connection.upstream.write(message) + } else if (this.dropReply(connection, message)) { + connection.client.destroy() + connection.upstream.destroy() + this.resolveDropped() + return + } else { + connection.client.write(message) + } + } + const remainder = buffer.subarray(offset) + if (direction === 'client') connection.clientBuffer = remainder + else connection.upstreamBuffer = remainder + } + + private observeRequest(connection: Connection, message: Buffer): void { + const header = this.header(message) + const command = this.command(message) + if (header === undefined || command === undefined || command.commitTransaction !== 1) return + const lsid = command.lsid + const txnNumber = command.txnNumber + if (lsid === undefined || txnNumber === undefined) + throw new Error('Mongo commit lacks session identity') + connection.commitRequests.add(header.requestId) + this.commits.push({ + requestId: header.requestId, + lsid: JSON.stringify(lsid), + txnNumber: String(txnNumber) + }) + } + + private dropReply(connection: Connection, message: Buffer): boolean { + if (this.dropped) return false + const header = this.header(message) + const command = this.command(message) + if ( + header === undefined || + command === undefined || + !connection.commitRequests.has(header.responseTo) + ) + return false + if (command.ok !== 1) return false + this.dropped = true + this.droppedSuccessfulCommitReply = true + return true + } + + private header(message: Buffer): { requestId: number; responseTo: number } | undefined { + if (message.readInt32LE(12) !== opMessage) return undefined + return { requestId: message.readInt32LE(4), responseTo: message.readInt32LE(8) } + } + + private command(message: Buffer): Record | undefined { + if (message.byteLength < 26 || message[20] !== 0) return undefined + const length = message.readInt32LE(21) + if (length < 5 || 21 + length > message.byteLength) return undefined + try { + return BSON.deserialize(message.subarray(21, 21 + length)) as Record + } catch { + return undefined + } + } +} diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts new file mode 100644 index 000000000..7254ca0d7 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts @@ -0,0 +1,564 @@ +import { createHash } from 'node:crypto' +import { setTimeout as delay } from 'node:timers/promises' +import { getAdmissionStorage } from '../../storage/AdmissionStorage.js' +import { bootstrapMongoOverlay } from '../../storage/mongo/MongoSchema.js' +import { MongoPayloadStore } from '../../storage/mongo/MongoPayloadStore.js' +import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' + +const bytes = async function* ( + value: Uint8Array, + split = value.byteLength +): AsyncIterable { + for (let index = 0; index < value.byteLength; index += split) + yield value.subarray(index, index + split) +} + +const digest = (value: Uint8Array): string => createHash('sha256').update(value).digest('hex') + +describe('MongoPayloadStore', () => { + let fixture: MongoReplicaFixture + let store: MongoPayloadStore + + beforeAll(async () => { + fixture = await createMongoReplicaFixture() + await bootstrapMongoOverlay(fixture.db, fixture.scope) + store = new MongoPayloadStore(fixture.db, fixture.scope) + }, 120000) + + afterAll(async () => { + await fixture.close() + }, 30000) + + test('stores BSON Binary inline and rejects corrupt stream declarations', async () => { + const content = Buffer.from('overlay-payload') + const hash = digest(content) + await expect( + store.publish({ + kind: 'locking-script', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content, 3) + }) + ).resolves.toMatchObject({ digest: hash }) + const row = await fixture.db.collection('overlay_payloads').findOne({ digest: hash }) + expect(row?.inlineData).toBeDefined() + await expect( + store.publish({ + kind: 'locking-script', + digest: digest(Buffer.from('different')), + byteLength: '1', + bytes: bytes(content) + }) + ).rejects.toThrow('declared length') + }) + + test('derives raw transaction identity from streamed bytes', async () => { + const content = Buffer.from('01000000000000000000', 'hex') + const hash = digest(content) + const expectedTxid = createHash('sha256') + .update(Buffer.from(hash, 'hex')) + .digest() + .reverse() + .toString('hex') + await expect( + store.publish({ + kind: 'raw-transaction', + digest: hash, + byteLength: String(content.byteLength), + txid: expectedTxid, + bytes: bytes(content, 1) + }) + ).resolves.toMatchObject({ txid: expectedTxid }) + await expect( + store.publish({ + kind: 'raw-transaction', + digest: digest(Buffer.from('unmatched')), + byteLength: String(content.byteLength), + txid: '00'.repeat(32), + bytes: bytes(content) + }) + ).rejects.toThrow() + }) + + test('requires actual ready-row guard in a caller transaction for references', async () => { + const content = Buffer.from('pinned') + const hash = digest(content) + await store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const reference = { + scope: fixture.scope, + payload: { kind: 'outbox-data' as const, digest: hash }, + ownerKind: 'pin' as const, + ownerId: 'test-pin', + slot: '0', + expiresAt: new Date(Date.now() + 60_000) + } + await expect(store.addReference(fixture.client.startSession(), reference)).rejects.toThrow( + 'active transaction' + ) + const session = fixture.client.startSession() + await session.withTransaction(async () => { + await store.addReference(session, reference) + }) + await session.endSession() + expect( + await fixture.db + .collection('overlay_payload_references') + .countDocuments({ ownerId: 'test-pin' }) + ).toBe(1) + }) + + test('streams individual payloads beyond BSON limits through GridFS', async () => { + const content = Buffer.alloc(16 * 1024 * 1024 + 17, 0x5a) + const hash = digest(content) + await store.publish({ + kind: 'locking-script', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content, 255 * 1024) + }) + const row = await fixture.db.collection('overlay_payloads').findOne({ digest: hash }) + expect(row?.fileId).toBeDefined() + expect(row?.inlineData).toBeUndefined() + const files = await fixture.db + .collection('overlayPayloads.files') + .countDocuments({ _id: row?.fileId, 'metadata.state': 'published' }) + expect(files).toBe(1) + + const rawHash = digest(Buffer.concat([content, Buffer.from([1])])) + await store.publish({ + kind: 'raw-transaction', + digest: rawHash, + byteLength: String(content.byteLength + 1), + bytes: bytes(Buffer.concat([content, Buffer.from([1])]), 255 * 1024) + }) + expect( + await fixture.db + .collection('overlay_payloads') + .countDocuments({ kind: 'raw-transaction', digest: rawHash, fileId: { $exists: true } }) + ).toBe(1) + }, 60000) + + test('cancels a stalled iterator and releases the reservation for a retry', async () => { + const content = Buffer.from('cancelled-then-retry') + const hash = digest(content) + const controller = new AbortController() + let returned = false + const stalled: AsyncIterable = { + [Symbol.asyncIterator]() { + return { + next: async () => await new Promise>(() => undefined), + return: async () => { + returned = true + return { done: true, value: undefined } + } + } + } + } + const pending = store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: stalled, + signal: controller.signal + }) + controller.abort(new Error('test cancellation')) + await expect(pending).rejects.toThrow('test cancellation') + expect(returned).toBe(true) + await expect( + store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + ).resolves.toMatchObject({ digest: hash }) + expect( + (await fixture.db.collection('overlay_payloads').findOne({ kind: 'outbox-data', digest: hash })) + ?.state + ).toBe('ready') + }) + + test('rejects an existing ready payload with a conflicting declared length', async () => { + const content = Buffer.from('same-content') + const hash = digest(content) + await store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + await expect( + store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength + 1), + bytes: bytes(content) + }) + ).rejects.toThrow('byte length') + }) + + test('recovers published staged files and retires corrupt staged files after an interrupted upload', async () => { + const content = Buffer.alloc(300 * 1024, 0x42) + const hash = digest(content) + await store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024) + }) + const payloads = fixture.db.collection('overlay_payloads') + const row = await payloads.findOne({ kind: 'outbox-data', digest: hash }) + expect(row?.fileId).toBeDefined() + await payloads.updateOne({ _id: row?._id }, [ + { + $set: { + state: 'uploading', + leaseUntil: { $dateSubtract: { startDate: '$$NOW', unit: 'second', amount: 1 } } + } + } + ]) + await fixture.db + .collection('overlayPayloads.files') + .updateOne({ _id: row?.fileId }, { $set: { 'metadata.state': 'staged' } }) + await store.recoverUploads() + expect((await payloads.findOne({ _id: row?._id }))?.state).toBe('ready') + + await payloads.updateOne({ _id: row?._id }, [ + { + $set: { + state: 'uploading', + leaseUntil: { $dateSubtract: { startDate: '$$NOW', unit: 'second', amount: 1 } } + } + } + ]) + await fixture.db.collection('overlayPayloads.chunks').deleteOne({ files_id: row?.fileId }) + await store.recoverUploads() + expect((await payloads.findOne({ _id: row?._id }))?.state).toBe('deleted') + }) + + test('serializes reference creation against a GC claim in both committed orders', async () => { + const content = Buffer.from('gc-order') + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const reference = { + scope: fixture.scope, + payload, + ownerKind: 'output' as const, + ownerId: 'gc-order', + slot: '0' + } + const first = fixture.client.startSession() + await first.withTransaction(async () => { + await store.addReference(first, reference) + }) + const second = fixture.client.startSession() + await second.withTransaction(async () => { + expect(await store.claimGarbage(second, payload)).toBe(false) + }) + await first.withTransaction(async () => { + await store.releaseReference(first, reference) + }) + await second.withTransaction(async () => { + expect(await store.claimGarbage(second, payload)).toBe(true) + }) + await expect( + first.withTransaction(async () => { + await store.addReference(first, reference) + }) + ).rejects.toThrow('not ready') + await Promise.all([first.endSession(), second.endSession()]) + }) + + test('uses Mongo server time when an explicit pin expires', async () => { + const content = Buffer.from('expiring-pin') + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const pin = { + scope: fixture.scope, + payload, + ownerKind: 'pin' as const, + ownerId: 'expiry', + slot: '0', + expiresAt: new Date(Date.now() + 60_000) + } + const session = fixture.client.startSession() + await session.withTransaction(async () => { + await store.addReference(session, pin) + }) + await session.withTransaction(async () => { + expect(await store.claimGarbage(session, payload)).toBe(false) + }) + await fixture.db.collection('overlay_payload_references').updateOne({ ownerId: 'expiry' }, [ + { + $set: { expiresAt: { $dateSubtract: { startDate: '$$NOW', unit: 'second', amount: 1 } } } + } + ]) + await session.withTransaction(async () => { + expect(await store.claimGarbage(session, payload)).toBe(true) + }) + await session.endSession() + }) + + test('uses majority journaled writes even when the caller client defaults to w:1', async () => { + const weak = await fixture.connect({ monitorCommands: true, writeConcern: { w: 1 } }) + const concerns: unknown[] = [] + weak.on('commandStarted', event => { + if (event.commandName === 'findAndModify' || event.commandName === 'update') + concerns.push(event.command.writeConcern) + }) + const weakStore = new MongoPayloadStore(weak.db(fixture.db.databaseName), fixture.scope) + const content = Buffer.from('explicit-write-concern') + await weakStore.publish({ + kind: 'outbox-data', + digest: digest(content), + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + expect(concerns).toContainEqual({ w: 'majority', j: true }) + await weak.close() + }) + + test('snapshots the constructor scope before asynchronous publication', async () => { + const mutableScope = { ...fixture.scope } + const isolated = new MongoPayloadStore(fixture.db, mutableScope) + mutableScope.network = 'mutated-network' + const content = Buffer.from('scope-snapshot') + await isolated.publish({ + kind: 'outbox-data', + digest: digest(content), + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + expect( + await fixture.db + .collection('overlay_payloads') + .countDocuments({ network: fixture.scope.network, digest: digest(content) }) + ).toBe(1) + expect( + await fixture.db + .collection('overlay_payloads') + .countDocuments({ network: 'mutated-network', digest: digest(content) }) + ).toBe(0) + }) + + test('live references from every non-pin owner class prevent a GC claim', async () => { + const content = Buffer.from('reference-guard') + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const ownerKinds = [ + 'transaction', + 'applied-history', + 'output', + 'gasp-graph', + 'gasp-node', + 'basm-job', + 'lookup-outbox', + 'propagation-outbox', + 'manifest' + ] as const + for (const ownerKind of ownerKinds) { + const reference = { + scope: fixture.scope, + payload, + ownerKind, + ownerId: `owner-${ownerKind}`, + slot: '0' + } + const session = fixture.client.startSession() + await session.withTransaction(async () => { + await store.addReference(session, reference) + }) + await session.withTransaction(async () => { + expect(await store.claimGarbage(session, payload)).toBe(false) + await store.releaseReference(session, reference) + }) + await session.endSession() + } + const session = fixture.client.startSession() + await session.withTransaction(async () => { + expect(await store.claimGarbage(session, payload)).toBe(true) + }) + await session.endSession() + expect(await store.finishGarbage(payload)).toBe(true) + }) + + test('does not advertise an admission-storage capability', () => { + expect(getAdmissionStorage(store)).toBeUndefined() + }) + + test('overlapping publishers of the same digest reuse one ready payload', async () => { + const content = Buffer.from('overlap-same-digest') + const hash = digest(content) + const publish = async (): Promise => + store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const [first, second] = await Promise.all([publish(), publish()]) + expect(first).toEqual(second) + expect( + await fixture.db + .collection('overlay_payloads') + .countDocuments({ kind: 'outbox-data', digest: hash, state: 'ready' }) + ).toBe(1) + }) + + test('identical bytes under a different kind keep separate metadata', async () => { + const content = Buffer.from('cross-kind-bytes') + const hash = digest(content) + const input = { + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content) + } + await store.publish({ ...input, kind: 'locking-script' }) + await store.publish({ ...input, kind: 'outbox-data', bytes: bytes(content) }) + expect(await fixture.db.collection('overlay_payloads').countDocuments({ digest: hash })).toBe(2) + }) + + test('recovers an upload that crashed after GridFS publication and before ready', async () => { + const content = Buffer.alloc(300 * 1024, 0x71) + const hash = digest(content) + const crashing = new MongoPayloadStore(fixture.db, fixture.scope, { + hooks: { + beforeReadyCas: () => { + throw new Error('crash after GridFS publication') + } + } + }) + await expect( + crashing.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024) + }) + ).rejects.toThrow('crash after GridFS publication') + const payloads = fixture.db.collection('overlay_payloads') + const row = await payloads.findOne({ kind: 'outbox-data', digest: hash }) + expect(row?.state).toBe('uploading') + expect(row?.fileId).toBeDefined() + await payloads.updateOne({ _id: row?._id }, [ + { + $set: { + leaseUntil: { $dateSubtract: { startDate: '$$NOW', unit: 'second', amount: 1 } } + } + } + ]) + await store.recoverUploads() + expect((await payloads.findOne({ _id: row?._id }))?.state).toBe('ready') + }) + + test('overlapping uncommitted GC and reference creation conflict on the payload row', async () => { + const content = Buffer.from('gc-overlap') + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const reference = { + scope: fixture.scope, + payload, + ownerKind: 'output' as const, + ownerId: 'gc-overlap', + slot: '0' + } + const pin = fixture.client.startSession() + const gc = fixture.client.startSession() + try { + pin.startTransaction() + await store.addReference(pin, reference) + gc.startTransaction() + const competing = store.claimGarbage(gc, payload) + await delay(25) + await pin.commitTransaction() + await expect(competing).rejects.toMatchObject({ code: 112 }) + } finally { + await Promise.allSettled([pin.abortTransaction(), gc.abortTransaction()]) + await Promise.all([pin.endSession(), gc.endSession()]) + } + }) + + test('physical GridFS deletion resumes after a crash between claim and finish', async () => { + const content = Buffer.alloc(300 * 1024, 0x63) + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024) + }) + const row = await fixture.db.collection('overlay_payloads').findOne({ digest: hash, kind: 'outbox-data' }) + expect(row?.fileId).toBeDefined() + const session = fixture.client.startSession() + await session.withTransaction(async () => { + expect(await store.claimGarbage(session, payload)).toBe(true) + }) + await session.endSession() + expect((await fixture.db.collection('overlay_payloads').findOne({ _id: row?._id }))?.state).toBe( + 'deleting' + ) + await store.recoverUploads() + expect((await fixture.db.collection('overlay_payloads').findOne({ _id: row?._id }))?.state).toBe( + 'deleted' + ) + expect( + await fixture.db.collection('overlayPayloads.files').countDocuments({ _id: row?.fileId }) + ).toBe(0) + expect(await store.finishGarbage(payload)).toBe(false) + }) + + test('shared manifest components pin the same payload against GC', async () => { + const content = Buffer.from('shared-ancestor') + const hash = digest(content) + const payload = { kind: 'raw-transaction' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const session = fixture.client.startSession() + await session.withTransaction(async () => { + await store.addManifestComponent(session, { + manifestId: 'manifest-a', + ordinal: '0', + kind: 'raw-transaction', + payload + }) + await store.addManifestComponent(session, { + manifestId: 'manifest-b', + ordinal: '0', + kind: 'raw-transaction', + payload + }) + }) + await session.withTransaction(async () => { + expect(await store.claimGarbage(session, payload)).toBe(false) + }) + await session.endSession() + }) +}) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoReadGuards.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoReadGuards.test.ts new file mode 100644 index 000000000..f79d3eb7f --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoReadGuards.test.ts @@ -0,0 +1,113 @@ +import { setTimeout as delay } from 'node:timers/promises' +import { + MongoReadGuardConflictError, + MongoReadGuards, + type MongoReadGuard +} from '../../storage/mongo/MongoReadGuards.js' +import { bootstrapMongoOverlay, MongoCollectionNames } from '../../storage/mongo/MongoSchema.js' +import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' + +describe('Mongo read guards', () => { + let fixture: MongoReplicaFixture + let guards: MongoReadGuards + const key = 'missing.output.0' + + const absentRead = (): MongoReadGuard => ({ + scope: fixture.scope, + key, + expectedVersion: null + }) + + beforeAll(async () => { + fixture = await createMongoReplicaFixture() + await bootstrapMongoOverlay(fixture.db, fixture.scope) + guards = new MongoReadGuards(fixture.db) + }, 90000) + + afterAll(async () => { + await fixture.close() + }) + + beforeEach(async () => { + await fixture.db.collection(MongoCollectionNames.readGuards).deleteMany({}) + await guards.initialize(fixture.scope, key) + }) + + test('requires the sentinel before an active transaction can make an absent predicate', async () => { + const session = fixture.client.startSession() + await expect(guards.check(session, absentRead())).rejects.toThrow( + 'Mongo read guards require an active transaction' + ) + await session.endSession() + }) + + test('reports a version mismatch as an explicit predicate conflict', async () => { + const session = fixture.client.startSession() + try { + session.startTransaction() + await guards.changeVersion(session, absentRead(), 'v1') + await session.commitTransaction() + } finally { + await session.endSession() + } + + const stale = fixture.client.startSession() + try { + stale.startTransaction() + await expect(guards.check(stale, absentRead())).rejects.toBeInstanceOf( + MongoReadGuardConflictError + ) + await stale.abortTransaction() + } finally { + await stale.endSession() + } + }) + + test('serializes a negative-read guard before a writer with an actual database write conflict', async () => { + const reader = fixture.client.startSession() + const writer = fixture.client.startSession() + try { + reader.startTransaction() + await guards.check(reader, absentRead()) + writer.startTransaction() + const competing = guards.changeVersion(writer, absentRead(), 'v1') + await delay(25) + await reader.commitTransaction() + await expect(competing).rejects.toMatchObject({ code: 112 }) + } finally { + await Promise.allSettled([reader.abortTransaction(), writer.abortTransaction()]) + await Promise.all([reader.endSession(), writer.endSession()]) + } + }) + + test('serializes a writer before a negative-read guard with an actual database write conflict', async () => { + const writer = fixture.client.startSession() + const reader = fixture.client.startSession() + try { + writer.startTransaction() + await guards.changeVersion(writer, absentRead(), 'v1') + reader.startTransaction() + const competing = guards.check(reader, absentRead()) + await delay(25) + await writer.commitTransaction() + await expect(competing).rejects.toMatchObject({ code: 112 }) + } finally { + await Promise.allSettled([reader.abortTransaction(), writer.abortTransaction()]) + await Promise.all([reader.endSession(), writer.endSession()]) + } + }) + + test('rejects malformed read inputs and bounded operation controls', async () => { + await expect(guards.initialize(fixture.scope, '', { timeoutMS: 1 })).rejects.toThrow( + 'Invalid Mongo read guard key' + ) + await expect(guards.initialize(fixture.scope, key, { timeoutMS: 30_001 })).rejects.toThrow( + 'Invalid Mongo read guard operation timeout' + ) + const controller = new AbortController() + controller.abort(new Error('cancelled')) + await expect( + guards.initialize(fixture.scope, key, { signal: controller.signal }) + ).rejects.toThrow('cancelled') + }) +}) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoReplicaFixture.ts b/packages/overlays/overlay/src/__tests/mongo/MongoReplicaFixture.ts new file mode 100644 index 000000000..a3db6689b --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoReplicaFixture.ts @@ -0,0 +1,146 @@ +import { randomUUID } from 'node:crypto' +import { once } from 'node:events' +import { setTimeout as delay } from 'node:timers/promises' +import { MongoClient, type Db, type Document, type MongoClientOptions } from 'mongodb' +import { MongoMemoryReplSet } from 'mongodb-memory-server' +import type { StorageScope } from '../../storage/AdmissionStorage.js' + +/** Owns only randomly named databases, ports and temporary mongod files. */ +export interface MongoReplicaFixture { + replicaSet: MongoMemoryReplSet + client: MongoClient + db: Db + uri: string + appName: string + scope: StorageScope + connect: (options?: MongoClientOptions, uri?: string) => Promise + failCommands: (data: Document, times?: number) => Promise + disableFailPoint: () => Promise + stepDown: () => Promise<{ previous: string; current: string }> + killPrimary: () => Promise<{ previous: string; current: string }> + close: () => Promise +} + +export async function createMongoReplicaFixture(): Promise { + const appName = `overlay-s02-${randomUUID()}` + const replicaSet = new MongoMemoryReplSet({ + binary: { version: '8.2.6' }, + replSet: { + name: `s02-${randomUUID()}`, + count: 3, + storageEngine: 'wiredTiger', + ip: '127.0.0.1', + args: [ + '--setParameter', + 'enableTestCommands=1', + '--wiredTigerCacheSizeGB', + '0.25' + ], + configSettings: { electionTimeoutMillis: 2000, heartbeatIntervalMillis: 500 } + }, + instanceOpts: [{ launchTimeout: 60000 }, { launchTimeout: 60000 }, { launchTimeout: 60000 }] + }) + try { + await replicaSet.start() + } catch (error) { + await replicaSet.stop() + throw error + } + const uri = replicaSet.getUri() + const clients: MongoClient[] = [] + const connect = async (options: MongoClientOptions = {}, connectionUri = uri) => { + const client = new MongoClient(connectionUri, { + appName, + serverSelectionTimeoutMS: 15000, + connectTimeoutMS: 5000, + heartbeatFrequencyMS: 500, + maxPoolSize: 8, + writeConcern: { w: 'majority', j: true }, + ...options + }) + clients.push(client) + await client.connect() + return client + } + let client: MongoClient + try { + client = await connect() + } catch (error) { + await Promise.allSettled(clients.map(item => item.close())) + await replicaSet.stop() + throw error + } + const db = client.db(`overlay_s02_${randomUUID().replaceAll('-', '')}`) + const primary = async (): Promise => { + const hello = await db.admin().command({ hello: 1 }, { timeoutMS: 15000 }) + if (typeof hello.primary !== 'string') throw new Error('Replica fixture has no primary') + return hello.primary + } + const waitForNewPrimary = async (previous: string): Promise => { + const deadline = Date.now() + 30000 + while (Date.now() < deadline) { + try { + const current = await primary() + if (current !== previous) return current + } catch { + // Election temporarily has no writable primary; the loop has a finite deadline. + } + await delay(100) + } + throw new Error('Replica fixture primary election timed out') + } + return { + replicaSet, + client, + db, + uri, + appName, + scope: { network: 'testnet', genesisHash: '11'.repeat(32), nodeId: 's02-node-a' }, + connect, + async failCommands(data, times = 1) { + await db.admin().command({ + configureFailPoint: 'failCommand', + mode: { times }, + data: { appName, ...data } + }) + }, + async disableFailPoint() { + await db.admin().command({ configureFailPoint: 'failCommand', mode: 'off' }) + }, + async stepDown() { + const previous = await primary() + const admin = await connect( + { appName: `${appName}-control`, directConnection: true }, + `mongodb://${previous}` + ) + try { + await admin.db('admin').command({ replSetStepDown: 10, force: true }, { timeoutMS: 15000 }) + } finally { + await admin.close() + } + return { previous, current: await waitForNewPrimary(previous) } + }, + async killPrimary() { + const previous = await primary() + const port = Number(previous.slice(previous.lastIndexOf(':') + 1)) + const server = replicaSet.servers.find(item => item.instanceInfo?.port === port) + const child = server?.instanceInfo?.instance.mongodProcess + if (child === undefined || child === null) { + throw new Error('Could not kill this fixture primary process') + } + const exited = once(child, 'close', { signal: AbortSignal.timeout(5000) }) + if (!child.kill('SIGKILL')) throw new Error('Could not kill this fixture primary process') + await exited + // memory-server 11.2 leaves this public handle set for an externally + // killed process; clear it only after observing that owned child's exit. + const instance = server?.instanceInfo?.instance + if (instance !== undefined) instance.mongodProcess = undefined + await server?.stop({ doCleanup: false }) + return { previous, current: await waitForNewPrimary(previous) } + }, + async close() { + await Promise.allSettled(clients.map(item => item.close())) + await replicaSet.stop() + } + } +} diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts new file mode 100644 index 000000000..1e5a965e0 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts @@ -0,0 +1,221 @@ +import { + bootstrapMongoOverlay, + decodeMongoUint64, + encodeMongoOutputIndex, + encodeMongoUint64, + MongoCollectionDefinitions, + MongoCollectionNames, + MongoGridFsBucketName, + mongoChainKey, + mongoNodeKey, + mongoRecordKey +} from '../../storage/mongo/MongoSchema.js' +import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' + +describe('Mongo schema codecs', () => { + test('root overlay exports do not include Mongo helpers', async () => { + const overlay = await import('../../../mod.js') + expect('bootstrapMongoOverlay' in overlay).toBe(false) + expect('MongoPayloadStore' in overlay).toBe(false) + expect('MongoTransactionRunner' in overlay).toBe(false) + }) + + test('uses framed values rather than separator-concatenated keys', () => { + expect(mongoRecordKey('a.b', 'c:d')).toBe('v1|3:a.b|3:c:d') + expect(mongoRecordKey('a', 'b.c')).not.toBe(mongoRecordKey('a.b', 'c')) + expect(mongoChainKey({ network: 'test.net', genesisHash: 'a'.repeat(64) })).toContain( + 'test.net' + ) + expect( + mongoNodeKey({ network: 'test.net', genesisHash: 'a'.repeat(64), nodeId: 'node.1' }) + ).toContain('node.1') + }) + + test.each(['', 'a\u0000b', 'a'.repeat(1025)])('rejects invalid key component %p', value => { + expect(() => mongoRecordKey(value)).toThrow('Invalid Mongo record key component') + }) + + test('encodes uint64 in lexically sortable exact form', () => { + expect(encodeMongoUint64('0')).toBe('00000000000000000000') + expect(encodeMongoUint64('18446744073709551615')).toBe('18446744073709551615') + expect(encodeMongoUint64('9') < encodeMongoUint64('10')).toBe(true) + expect(decodeMongoUint64('00000000000000000009')).toBe('9') + expect(() => decodeMongoUint64('18446744073709551616')).toThrow('Invalid Mongo uint64') + expect(() => decodeMongoUint64('9')).toThrow('Invalid Mongo uint64') + }) + + test('keeps output indexes in the wire uint32 domain', () => { + expect(encodeMongoOutputIndex('4294967295')).toBe('4294967295') + expect(() => encodeMongoOutputIndex('4294967296')).toThrow('Invalid storage output index') + }) + + test('defines every Overlay-owned collection with strict versioned validators', () => { + expect(MongoCollectionDefinitions.map(definition => definition.name)).toEqual( + Object.values(MongoCollectionNames) + ) + for (const definition of MongoCollectionDefinitions) { + expect(definition.validator.$and[0].$jsonSchema.additionalProperties).toBe(false) + expect(definition.validator.$and[0].$jsonSchema.required).toContain('schemaVersion') + expect(definition.indexes.length).toBeGreaterThan(0) + } + }) +}) + +describe('Mongo schema bootstrap', () => { + let fixture: MongoReplicaFixture + + beforeAll(async () => { + fixture = await createMongoReplicaFixture() + }, 90000) + + afterAll(async () => { + await fixture.close() + }) + + test('creates and revalidates the complete versioned schema on a replica set', async () => { + await expect(bootstrapMongoOverlay(fixture.db, fixture.scope)).resolves.toMatchObject({ + topology: 'replica-set', + schemaVersion: 1, + scope: fixture.scope, + collections: Object.values(MongoCollectionNames) + }) + await expect(bootstrapMongoOverlay(fixture.db, fixture.scope)).resolves.toMatchObject({ + topology: 'replica-set', + schemaVersion: 1 + }) + expect( + await fixture.db.listCollections({ name: `${MongoGridFsBucketName}.files` }).hasNext() + ).toBe(true) + expect( + await fixture.db.listCollections({ name: `${MongoGridFsBucketName}.chunks` }).hasNext() + ).toBe(true) + }, 30000) + + test('refuses an existing collection whose validator is not the Overlay schema', async () => { + const database = fixture.client.db(`overlay_s02_drift_${Date.now()}`) + await database.createCollection(MongoCollectionNames.outputs) + await expect(bootstrapMongoOverlay(database, fixture.scope)).rejects.toThrow( + `Incompatible Mongo Overlay validator for ${MongoCollectionNames.outputs}` + ) + }) + + test('enforces the uint64 domain in direct payload, operation, output, and history writes', async () => { + const now = new Date() + const base = { schemaVersion: 1, ...fixture.scope, createdAt: now, updatedAt: now } + const chainBase = { + schemaVersion: 1, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash, + createdAt: now, + updatedAt: now + } + const payload = (id: string, byteLength: string, digest: string) => ({ + _id: id, + ...chainBase, + kind: 'raw-transaction', + digest, + byteLength, + state: 'uploading', + guard: 'guard', + ownerNodeId: fixture.scope.nodeId, + ownerId: 'owner', + fencingToken: encodeMongoUint64('0'), + leaseUntil: now + }) + await expect( + fixture.db + .collection(MongoCollectionNames.payloads) + .insertOne(payload('payload-invalid', '99999999999999999999', 'aa'.repeat(32))) + ).rejects.toThrow() + await fixture.db + .collection(MongoCollectionNames.payloads) + .insertMany([ + payload('payload-unsafe-integer', encodeMongoUint64('9007199254740993'), 'ab'.repeat(32)), + payload('payload-uint64-max', encodeMongoUint64('18446744073709551615'), 'ac'.repeat(32)) + ]) + await expect( + fixture.db.collection(MongoCollectionNames.submissionOperations).insertOne({ + _id: 'operation-invalid', + ...base, + operationId: 'operation-invalid', + semanticDigest: 'bb'.repeat(32), + txid: 'cc'.repeat(32), + state: 'aborted', + attemptId: '00000000-0000-4000-8000-000000000001', + leaseOwner: '00000000-0000-4000-8000-000000000002', + leaseToken: '99999999999999999999', + leaseUntil: now, + guard: '00000000-0000-4000-8000-000000000003' + }) + ).rejects.toThrow() + await expect( + fixture.db.collection(MongoCollectionNames.outputs).insertOne({ + _id: 'output-invalid', + ...base, + topic: 'topic', + txid: 'dd'.repeat(32), + outputIndex: '0', + satoshis: '99999999999999999999', + score: encodeMongoUint64('0'), + scriptPayloadId: 'payload', + scriptOffset: encodeMongoUint64('0'), + scriptByteLength: encodeMongoUint64('0'), + state: 'unspent', + version: 'version' + }) + ).rejects.toThrow() + await expect( + fixture.db.collection(MongoCollectionNames.topicGenerations).insertOne({ + _id: 'history-invalid', + ...base, + topic: 'topic', + chainEpoch: '99999999999999999999', + topicHistoryGeneration: encodeMongoUint64('0'), + policyId: 'policy' + }) + ).rejects.toThrow() + }) + + test('rejects direct documents without required audit dates', async () => { + await expect( + fixture.db.collection(MongoCollectionNames.payloads).insertOne({ + _id: 'payload-without-audit-dates', + schemaVersion: 1, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash, + kind: 'raw-transaction', + digest: 'ef'.repeat(32), + byteLength: encodeMongoUint64('0'), + state: 'uploading', + guard: 'guard', + ownerNodeId: fixture.scope.nodeId, + ownerId: 'owner', + fencingToken: encodeMongoUint64('0'), + leaseUntil: new Date() + }) + ).rejects.toThrow() + }) + + test('rejects a pre-existing partial index whose predicate has drifted', async () => { + const definition = MongoCollectionDefinitions.find( + item => item.name === MongoCollectionNames.payloadReferences + ) + if (definition === undefined) throw new Error('Missing payload reference schema definition') + const database = fixture.client.db(`overlay_s02_index_drift_${Date.now()}`) + await database.createCollection(MongoCollectionNames.payloadReferences, { + validator: definition.validator, + validationLevel: 'strict', + validationAction: 'error', + collation: { locale: 'simple' } + }) + await database + .collection(MongoCollectionNames.payloadReferences) + .createIndex( + { expiresAt: 1 }, + { name: 'pin_expiry', partialFilterExpression: { ownerKind: 'transaction' } } + ) + await expect(bootstrapMongoOverlay(database, fixture.scope)).rejects.toThrow( + `Incompatible Mongo Overlay index for ${MongoCollectionNames.payloadReferences}:pin_expiry` + ) + }) +}) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts new file mode 100644 index 000000000..9519d4c85 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts @@ -0,0 +1,314 @@ +import { randomUUID } from 'node:crypto' +import { setTimeout as delay } from 'node:timers/promises' +import { Binary, type CommandStartedEvent, type Document } from 'mongodb' +import { + admissionSemanticDigest, + getAdmissionStorage, + type AdmissionIdentity +} from '../../storage/AdmissionStorage.js' +import { bootstrapMongoOverlay, encodeMongoUint64, MongoCollectionNames, mongoRecordKey } from '../../storage/mongo/MongoSchema.js' +import { MongoTransactionRunner, type MongoTransactionRequest } from '../../storage/mongo/MongoTransactionRunner.js' +import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' + +describe('Mongo transaction boundary on three data-bearing WiredTiger members', () => { + let fixture: MongoReplicaFixture + let runner: MongoTransactionRunner + const runners: MongoTransactionRunner[] = [] + const commands: CommandStartedEvent[] = [] + + function request(operationId = randomUUID()): MongoTransactionRequest { + const identity: AdmissionIdentity = { scope: fixture.scope, txid: '22'.repeat(32), mode: 'live', contextDigest: '33'.repeat(32), topics: [{ topic: 'topic.a', policyId: 'v1' }] } + const semanticDigest = admissionSemanticDigest(identity) + return { identity, key: { scope: fixture.scope, operationId, semanticDigest }, receipt: { operationId, semanticDigest, durability: 'atomic-local', steak: '{ "topic.a" : { "outputsToAdmit": [0] } }\n', indexes: [{ target: 'lookup', state: 'pending' }], propagation: 'pending' } } + } + + function operationId(input: MongoTransactionRequest): string { + return mongoRecordKey('operation', fixture.scope.network, fixture.scope.genesisHash, fixture.scope.nodeId, input.key.operationId) + } + + beforeAll(async () => { + fixture = await createMongoReplicaFixture() + await bootstrapMongoOverlay(fixture.db, fixture.scope) + // Driver core calls retain their own retry policy. Disable it in this + // fixture client so each failpoint corresponds to one runner commit attempt. + const client = await fixture.connect({ monitorCommands: true, maxAdaptiveRetries: 0 }) + client.on('commandStarted', event => commands.push(event)) + fixture.db = client.db(fixture.db.databaseName) + runner = new MongoTransactionRunner(client.db(fixture.db.databaseName), fixture.scope, { maxCommitAttempts: 2 }) + runners.push(runner) + }, 90000) + + afterEach(async () => { + await fixture.disableFailPoint() + }) + + afterAll(async () => { + await Promise.all(runners.map(async item => await item.close())) + await fixture.close() + }, 60000) + + test('does not advertise an admission-storage capability', () => { + expect(getAdmissionStorage(runner)).toBeUndefined() + }) + + test('saves exact receipt with effects and replays it after runner restart', async () => { + const input = request() + let bodies = 0 + const result = await runner.run(input, async context => { + bodies += 1 + await fixture.db.collection('test_effects').insertOne({ operationId: input.key.operationId, value: '18446744073709551615' }, context.options()) + }) + expect(result).toEqual({ state: 'committed', receipt: input.receipt }) + const replacement = new MongoTransactionRunner(fixture.db, fixture.scope) + runners.push(replacement) + expect(await replacement.run(input, async () => { bodies += 1 })).toEqual(result) + expect(await replacement.reconcile(input.key)).toEqual(result) + expect(bodies).toBe(1) + const stored = await fixture.db.collection(MongoCollectionNames.submissionOperations).findOne({ _id: operationId(input) }) + expect(stored?.receipt).toBeInstanceOf(Binary) + expect(await fixture.db.collection('test_effects').countDocuments({ operationId: input.key.operationId })).toBe(1) + }) + + test('rejects conflicting semantics and wrong scope without running a body', async () => { + const input = request() + await runner.run(input, async () => {}) + const changed = structuredClone(input) + changed.identity.contextDigest = '44'.repeat(32) + changed.key.semanticDigest = admissionSemanticDigest(changed.identity) + changed.receipt.semanticDigest = changed.key.semanticDigest + const body = jest.fn(async () => {}) + expect(await runner.run(changed, body)).toEqual({ state: 'rejected', code: 'digest-mismatch' }) + changed.key.scope.nodeId = 'different-node' + await expect(runner.run(changed, body)).rejects.toThrow('scope') + expect(body).not.toHaveBeenCalled() + }) + + test('aborts all body effects on an ordinary error and permits a later fresh attempt', async () => { + const input = request() + await expect(runner.run(input, async context => { + await fixture.db.collection('test_effects').insertOne({ operationId: input.key.operationId }, context.options()) + throw new Error('decision failed') + })).rejects.toThrow('decision failed') + expect(await fixture.db.collection('test_effects').countDocuments({ operationId: input.key.operationId })).toBe(0) + expect(await runner.reconcile(input.key)).toEqual({ state: 'aborted' }) + expect((await runner.run(input, async () => {})).state).toBe('committed') + }) + + test('a real transient server error permits a bounded fresh body and no partial effect', async () => { + const input = request() + let bodies = 0 + const result = await runner.run(input, async context => { + bodies += 1 + if (bodies === 1) await fixture.failCommands({ failCommands: ['insert'], errorCode: 112, errorLabels: ['TransientTransactionError'] }) + await fixture.db.collection('test_effects').insertOne({ operationId: input.key.operationId, body: bodies }, context.options()) + }) + expect(result.state).toBe('committed') + expect(bodies).toBe(2) + expect(await fixture.db.collection('test_effects').find({ operationId: input.key.operationId }).toArray()).toEqual([expect.objectContaining({ body: 2 })]) + }) + + test('unknown commit exhausts finitely, never reruns body, and resumes the same session and transaction number', async () => { + const input = request() + const start = commands.length + let bodies = 0 + await fixture.failCommands({ failCommands: ['commitTransaction'], errorCode: 91, errorLabels: ['UnknownTransactionCommitResult'] }, 2) + const uncertain = await runner.run(input, async context => { + bodies += 1 + await fixture.db.collection('test_effects').insertOne({ operationId: input.key.operationId }, context.options()) + }) + expect(uncertain.state).toBe('pending') + expect(await runner.run(input, async () => { bodies += 1 })).toEqual(uncertain) + await fixture.disableFailPoint() + const result = await runner.reconcile(input.key, uncertain.state === 'pending' ? uncertain.attemptId : undefined) + expect(result).toEqual({ state: 'committed', receipt: input.receipt }) + expect(bodies).toBe(1) + const commits = commands.slice(start).filter(event => event.commandName === 'commitTransaction') + expect(commits.length).toBe(3) + expect(new Set(commits.map(event => `${String(event.command.lsid.id)}:${String(event.command.txnNumber)}`)).size).toBe(1) + }, 15000) + + test('unknown label wins when a server error also carries a transient label', async () => { + const input = request() + let bodies = 0 + await fixture.failCommands({ failCommands: ['commitTransaction'], errorCode: 91, errorLabels: ['UnknownTransactionCommitResult', 'TransientTransactionError'] }, 2) + const result = await runner.run(input, async () => { bodies += 1 }) + expect(result.state).toBe('pending') + expect(bodies).toBe(1) + await fixture.disableFailPoint() + expect((await runner.reconcile(input.key)).state).toBe('committed') + }, 15000) + + test('missing operation is pending, and expired orphan abort requires a successful majority row write', async () => { + const input = request() + expect(await runner.reconcile(input.key, 'unknown-attempt')).toEqual({ state: 'pending', attemptId: 'unknown-attempt' }) + const attemptId = randomUUID() + await fixture.db.collection(MongoCollectionNames.submissionOperations).insertOne({ _id: operationId(input), schemaVersion: 1, ...fixture.scope, operationId: input.key.operationId, semanticDigest: input.key.semanticDigest, txid: input.identity.txid, state: 'pending', attemptId, leaseOwner: randomUUID(), leaseToken: encodeMongoUint64('9007199254740993'), leaseUntil: new Date(0), guard: randomUUID(), createdAt: new Date(), updatedAt: new Date() }) + expect(await runner.reconcile(input.key, attemptId)).toEqual({ state: 'aborted' }) + const row = await fixture.db.collection(MongoCollectionNames.submissionOperations).findOne({ _id: operationId(input) }) + expect(row?.state).toBe('aborted') + expect((await runner.run(input, async () => {})).state).toBe('committed') + }) + + test('concurrent same-key calls do not fork the body', async () => { + const input = request() + let entered!: () => void + const started = new Promise(resolve => { entered = resolve }) + let resume!: () => void + const barrier = new Promise(resolve => { resume = resolve }) + let bodies = 0 + const first = runner.run(input, async () => { bodies += 1; entered(); await barrier }) + await started + const second = await runner.run(input, async () => { bodies += 1 }) + expect(second.state).toBe('pending') + expect(await runner.reconcile(input.key)).toEqual(second) + resume() + expect((await first).state).toBe('committed') + expect(bodies).toBe(1) + }) + + test('cancellation fences subsequent operations and rolls back the body', async () => { + const input = request() + const controller = new AbortController() + let lateOptions: (() => unknown) | undefined + await expect(runner.run(input, async context => { + lateOptions = context.options + await fixture.db.collection('test_effects').insertOne({ operationId: input.key.operationId }, context.options()) + controller.abort(new Error('cancelled fixture')) + await delay(5) + context.options() + }, { signal: controller.signal })).rejects.toThrow('cancelled fixture') + expect(() => lateOptions?.()).toThrow('no longer active') + expect(await fixture.db.collection('test_effects').countDocuments({ operationId: input.key.operationId })).toBe(0) + }) + + test('snapshots caller identity before asynchronous storage work', async () => { + const input = structuredClone(request()) + const original = structuredClone(input) + const pending = runner.run(input, async () => {}) + input.key.operationId = 'mutated-operation' + input.identity.txid = '99'.repeat(32) + input.key.scope.nodeId = 'mutated-node' + input.receipt.steak = '{}' + expect(await pending).toEqual({ state: 'committed', receipt: original.receipt }) + const row = await fixture.db.collection(MongoCollectionNames.submissionOperations).findOne({ _id: operationId(original) }) + expect(row).toMatchObject({ operationId: original.key.operationId, txid: original.identity.txid, nodeId: fixture.scope.nodeId }) + }) + + test('the retained-session capacity also bounds simultaneous different-key claims', async () => { + const limited = new MongoTransactionRunner(fixture.db, fixture.scope, { maxRetainedSessions: 1 }) + runners.push(limited) + let resume!: () => void + const barrier = new Promise(resolve => { resume = resolve }) + let entered!: () => void + const started = new Promise(resolve => { entered = resolve }) + let bodies = 0 + const body = async () => { bodies += 1; entered(); await barrier } + const calls = [limited.run(request(), body), limited.run(request(), body)] + const rejected = Promise.any(calls.map(async call => { + try { await call; throw new Error('Unexpected success before barrier') } catch (error) { return error } + })) + await started + expect(await rejected).toEqual(expect.objectContaining({ message: 'Mongo unresolved transaction capacity reached' })) + expect(bodies).toBe(1) + resume() + const results = await Promise.allSettled(calls) + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(1) + }) + + test('server transient failures stop at the configured body budget', async () => { + const input = request() + let bodies = 0 + await expect(runner.run(input, async context => { + bodies += 1 + await fixture.failCommands({ failCommands: ['insert'], errorCode: 112, errorLabels: ['TransientTransactionError'] }) + await fixture.db.collection('test_effects').insertOne({ operationId: input.key.operationId }, context.options()) + })).rejects.toMatchObject({ code: 112 }) + expect(bodies).toBe(3) + expect(await runner.reconcile(input.key)).toEqual({ state: 'aborted' }) + }) + + test('write CSOT bounds a server-blocked body and pre-cancellation starts no body', async () => { + const input = request() + const controller = new AbortController() + controller.abort(new Error('before claim')) + const body = jest.fn(async () => {}) + await expect(runner.run(input, body, { signal: controller.signal })).rejects.toThrow('before claim') + expect(body).not.toHaveBeenCalled() + const started = performance.now() + await expect(runner.run(input, async context => { + await fixture.failCommands({ failCommands: ['insert'], blockConnection: true, blockTimeMS: 500 }) + await fixture.db.collection('test_effects').insertOne({ operationId: input.key.operationId }, context.options()) + }, { timeoutMS: 150 })).rejects.toThrow() + expect(performance.now() - started).toBeLessThan(2500) + expect(await fixture.db.collection('test_effects').countDocuments({ operationId: input.key.operationId })).toBe(0) + }) + + test('expired orphan CAS wins against a stale transaction snapshot', async () => { + const input = request() + const attemptId = randomUUID() + const collection = fixture.db.collection(MongoCollectionNames.submissionOperations) + await collection.insertOne({ _id: operationId(input), schemaVersion: 1, ...fixture.scope, operationId: input.key.operationId, semanticDigest: input.key.semanticDigest, txid: input.identity.txid, state: 'pending', attemptId, leaseOwner: randomUUID(), leaseToken: encodeMongoUint64('1'), leaseUntil: new Date(0), guard: randomUUID(), createdAt: new Date(), updatedAt: new Date() }) + const session = fixture.db.client.startSession() + try { + session.startTransaction({ readConcern: { level: 'snapshot' }, writeConcern: { w: 'majority', j: true } }) + expect((await collection.findOne({ _id: operationId(input) }, { session }))?.state).toBe('pending') + expect(await runner.reconcile(input.key, attemptId)).toEqual({ state: 'aborted' }) + await expect(collection.updateOne({ _id: operationId(input), state: 'pending', attemptId }, { $set: { guard: randomUUID() } }, { session, timeoutMS: 1000 })).rejects.toMatchObject({ code: 112 }) + } finally { + if (session.inTransaction()) await session.abortTransaction() + await session.endSession() + } + }) + + test('a committed receipt wins against overlapping expired-orphan reconciliation', async () => { + const input = request() + const attemptId = randomUUID() + const collection = fixture.db.collection(MongoCollectionNames.submissionOperations) + await collection.insertOne({ _id: operationId(input), schemaVersion: 1, ...fixture.scope, operationId: input.key.operationId, semanticDigest: input.key.semanticDigest, txid: input.identity.txid, state: 'pending', attemptId, leaseOwner: randomUUID(), leaseToken: encodeMongoUint64('1'), leaseUntil: new Date(0), guard: randomUUID(), createdAt: new Date(), updatedAt: new Date() }) + const session = fixture.db.client.startSession() + let listener: ((event: CommandStartedEvent) => void) | undefined + try { + session.startTransaction({ readConcern: { level: 'snapshot' }, writeConcern: { w: 'majority', j: true } }) + await collection.updateOne({ _id: operationId(input), state: 'pending', attemptId }, { $set: { guard: randomUUID(), state: 'committed', receipt: new Binary(Buffer.from(JSON.stringify(input.receipt))) } }, { session }) + const attemptedCas = new Promise(resolve => { + listener = event => { + if (event.commandName === 'findAndModify' && event.command.query.attemptId === attemptId) resolve() + } + fixture.db.client.on('commandStarted', listener) + }) + const reconciliation = runner.reconcile(input.key, attemptId) + await attemptedCas + await session.commitTransaction({ timeoutMS: 5000 }) + expect(await reconciliation).toEqual({ state: 'committed', receipt: input.receipt }) + } finally { + if (listener !== undefined) fixture.db.client.off('commandStarted', listener) + if (session.inTransaction()) await session.abortTransaction() + await session.endSession() + } + }) + + test('actual primary stepdown aborts the old body and commits a fresh transaction', async () => { + const input = request() + let bodies = 0 + const result = await runner.run(input, async context => { + bodies += 1 + if (bodies === 1) await fixture.stepDown() + await fixture.db.collection('test_effects').insertOne({ operationId: input.key.operationId, body: bodies }, context.options()) + }, { timeoutMS: 40000 }) + expect(result.state).toBe('committed') + expect(bodies).toBe(2) + }, 60000) + + test('majority receipt and effects survive an actual SIGKILL of the acknowledged primary', async () => { + const input = request() + expect((await runner.run(input, async context => { + await fixture.db.collection('test_effects').insertOne({ operationId: input.key.operationId }, context.options()) + }, { timeoutMS: 15000 })).state).toBe('committed') + const election = await fixture.killPrimary() + expect(election.current).not.toBe(election.previous) + const restarted = new MongoTransactionRunner(fixture.db, fixture.scope) + runners.push(restarted) + expect(await restarted.reconcile(input.key, undefined, { timeoutMS: 15000 })).toEqual({ state: 'committed', receipt: input.receipt }) + expect(await fixture.db.collection('test_effects').countDocuments({ operationId: input.key.operationId })).toBe(1) + }, 60000) +}) diff --git a/packages/overlays/overlay/src/storage/mongo.ts b/packages/overlays/overlay/src/storage/mongo.ts new file mode 100644 index 000000000..d73a111ef --- /dev/null +++ b/packages/overlays/overlay/src/storage/mongo.ts @@ -0,0 +1,5 @@ +/** Opt-in Mongo primitives. Importing the root Overlay entry does not load this driver. */ +export * from './mongo/MongoSchema.js' +export * from './mongo/MongoPayloadStore.js' +export * from './mongo/MongoReadGuards.js' +export * from './mongo/MongoTransactionRunner.js' diff --git a/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts b/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts new file mode 100644 index 000000000..ab899ecd2 --- /dev/null +++ b/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts @@ -0,0 +1,889 @@ +import { createHash, randomUUID } from 'node:crypto' +import { once } from 'node:events' +import { setTimeout as delay } from 'node:timers/promises' +import { + BSON, + Binary, + GridFSBucket, + ObjectId, + type ClientSession, + type Collection, + type Db, + type Document +} from 'mongodb' +import { + MongoCollectionNames, + MongoGridFsBucketName, + type MongoChainScope, + mongoChainKey, + mongoNodeKey, + mongoRecordKey, + encodeMongoUint64, + decodeMongoUint64 +} from './MongoSchema.js' + +/** Content types intentionally do not encode a caller's ownership domain. */ +export type MongoPayloadKind = + 'raw-transaction' | 'merkle-path' | 'beef-manifest' | 'locking-script' | 'outbox-data' + +export type MongoPayloadReferenceOwnerKind = + | 'transaction' + | 'applied-history' + | 'output' + | 'gasp-graph' + | 'gasp-node' + | 'basm-job' + | 'lookup-outbox' + | 'propagation-outbox' + | 'manifest' + | 'pin' + +export interface MongoPayloadInput { + kind: MongoPayloadKind + /** SHA-256 of the canonical bytes, provided before upload to obtain a fenced reservation. */ + digest: string + bytes: AsyncIterable + /** Canonical decimal uint64. The stream must contain exactly this many bytes. */ + byteLength: string + /** Required when the caller has already derived the raw transaction identity. */ + txid?: string + signal?: AbortSignal +} + +export interface MongoPayloadRef { + kind: MongoPayloadKind + digest: string + byteLength: string + /** Bitcoin display-order txid, only for raw transaction payloads. */ + txid?: string +} + +export interface MongoPayloadReference { + scope: MongoChainScope & { nodeId: string } + payload: Pick + ownerKind: MongoPayloadReferenceOwnerKind + ownerId: string + slot: string + /** Explicit pins are the sole reference class permitted to expire. */ + expiresAt?: Date +} + +/** A manifest is an ordered list of independent content-addressed components. */ +export interface MongoManifestComponent { + manifestId: string + ordinal: string + kind: MongoPayloadKind + payload: Pick +} + +export interface MongoPayloadStoreOptions { + /** Kept deliberately far below Mongo's document cap. */ + inlineCeilingBytes?: number + /** A GridFS chunk must remain below the BSON document cap with ample overhead. */ + gridFsChunkBytes?: number + uploadLeaseMs?: number + /** Application-level byte bound; BSON/GridFS are not an admission resource limit. */ + maxPayloadBytes?: bigint + now?: () => Date + /** Narrow I/O seam used only to exercise crash boundaries. */ + hooks?: Partial< + Record< + 'afterGridFsUploaded' | 'afterGridFsPublished' | 'beforeReadyCas' | 'afterDeleteClaim', + () => Promise | void + > + > +} + +/** Per-operation cancellation/deadline controls supplied by the admission body. */ +export interface MongoPayloadOperationOptions { + timeoutMS?: number + signal?: AbortSignal +} + +type PayloadState = 'uploading' | 'ready' | 'deleting' | 'deleted' + +interface PayloadDocument extends Document { + _id: string + network: string + genesisHash: string + kind: MongoPayloadKind + digest: string + byteLength: string + state: PayloadState + guard: string + ownerNodeId: string + ownerId: string + fencingToken: string + leaseUntil: Date + createdAt: Date + updatedAt: Date + inlineData?: Binary + fileId?: ObjectId + retiredFileId?: ObjectId + uploadId?: string +} + +interface ReferenceDocument extends Document { + _id: string + payloadId: string + ownerKind: MongoPayloadReferenceOwnerKind + expiresAt?: Date +} + +const MAX_UINT64 = (BigInt(1) << BigInt(64)) - BigInt(1) +const HEX_256 = /^[0-9a-f]{64}$/ + +function duplicateKey(error: unknown): boolean { + return typeof error === 'object' && error !== null && 'code' in error && error.code === 11000 +} + +/** + * Content-addressed blob primitives for the Mongo adapter. The store deliberately + * has no Engine/Storage methods: S03 combines its reference operations with the + * owner record in an admission transaction. + */ +export class MongoPayloadStore { + private readonly inlineCeiling: number + private readonly gridFsChunkBytes: number + private readonly uploadLeaseMs: number + private readonly maxPayloadBytes: bigint + private readonly now: () => Date + private readonly bucket: GridFSBucket + private readonly scope: MongoChainScope & { nodeId: string } + private readonly writeConcern = { w: 'majority' as const, j: true } + + public constructor( + private readonly db: Db, + scope: MongoChainScope & { nodeId: string }, + private readonly options: MongoPayloadStoreOptions = {} + ) { + this.scope = { network: scope.network, genesisHash: scope.genesisHash, nodeId: scope.nodeId } + this.inlineCeiling = options.inlineCeilingBytes ?? 256 * 1024 + this.gridFsChunkBytes = options.gridFsChunkBytes ?? 255 * 1024 + this.uploadLeaseMs = options.uploadLeaseMs ?? 60_000 + this.maxPayloadBytes = options.maxPayloadBytes ?? BigInt(64 * 1024 * 1024) + this.now = options.now ?? (() => new Date()) + if ( + !Number.isSafeInteger(this.inlineCeiling) || + this.inlineCeiling < 1 || + this.inlineCeiling > 1024 * 1024 + ) + throw new Error('Invalid Mongo payload inline ceiling') + if ( + !Number.isSafeInteger(this.gridFsChunkBytes) || + this.gridFsChunkBytes < 1 || + this.gridFsChunkBytes > 255 * 1024 + ) + throw new Error('Invalid Mongo GridFS chunk size') + if (!Number.isSafeInteger(this.uploadLeaseMs) || this.uploadLeaseMs < 1) + throw new Error('Invalid Mongo upload lease') + if (this.maxPayloadBytes < BigInt(1) || this.maxPayloadBytes > MAX_UINT64) + throw new Error('Invalid Mongo payload byte bound') + this.bucket = new GridFSBucket(db, { + bucketName: MongoGridFsBucketName, + chunkSizeBytes: this.gridFsChunkBytes, + writeConcern: this.writeConcern + }) + } + + public async publish(input: MongoPayloadInput): Promise { + input = { ...input } + this.validateInput(input) + const payloadId = mongoRecordKey(mongoChainKey(this.scope), input.kind, input.digest) + const payloads = this.db.collection(MongoCollectionNames.payloads) + const reserved = await this.reserveUpload(payloads, payloadId, input) + if (reserved.ready) return this.refFor(input, reserved.record) + const ownerId = reserved.record.ownerId + const fence = reserved.record.fencingToken + + let fileId: ObjectId | undefined + try { + const streamed = await this.streamInput(input, payloadId, ownerId, fence) + this.throwIfAborted(input.signal) + fileId = streamed.fileId + if (streamed.digest !== input.digest || streamed.length !== BigInt(input.byteLength)) + throw new Error('Mongo payload digest or declared length mismatch') + if ( + input.kind === 'raw-transaction' && + input.txid !== undefined && + input.txid !== streamed.txid + ) + throw new Error('Mongo raw transaction id does not match bytes') + const binary = + streamed.inline === undefined ? undefined : new Binary(Buffer.concat(streamed.inline)) + if (binary !== undefined && !this.fitsInline(payloadId, input, binary, this.now())) + throw new Error('Mongo payload inline BSON document exceeds safety ceiling') + if (fileId !== undefined) { + const staged = await payloads.updateOne( + { _id: payloadId, state: 'uploading', ownerId, guard: ownerId }, + { $set: { fileId, updatedAt: this.now() } }, + { writeConcern: this.writeConcern } + ) + if (staged.matchedCount !== 1) + throw new Error('Mongo payload upload lease was fenced before staging') + await this.verifyGridFs(fileId, input, input.signal) + this.throwIfAborted(input.signal) + await this.options.hooks?.afterGridFsUploaded?.() + this.throwIfAborted(input.signal) + const published = await this.db.collection(`${MongoGridFsBucketName}.files`).updateOne( + { + _id: fileId, + 'metadata.ownerId': ownerId, + 'metadata.fencingToken': fence, + 'metadata.state': 'staged' + }, + { $set: { 'metadata.state': 'published' } }, + { writeConcern: this.writeConcern } + ) + if (published.matchedCount !== 1) throw new Error('Mongo GridFS staged file was lost') + await this.options.hooks?.afterGridFsPublished?.() + } + await this.options.hooks?.beforeReadyCas?.() + this.throwIfAborted(input.signal) + const ready = await payloads.findOneAndUpdate( + { + _id: payloadId, + state: 'uploading', + ownerId, + guard: ownerId, + $expr: { $gt: ['$leaseUntil', '$$NOW'] } + }, + { + $set: { + state: 'ready', + updatedAt: this.now(), + ...(fileId === undefined ? { inlineData: binary } : { fileId }) + }, + $unset: { uploadId: '' } + }, + { returnDocument: 'after', writeConcern: this.writeConcern } + ) + if (ready === null) + throw new Error('Mongo payload upload lease was fenced before publication') + return this.refFor(input, ready) + } catch (error) { + // Never remove a file by payload identity: a newer fenced owner may have published it. + if (fileId !== undefined) { + const published = await this.db.collection(`${MongoGridFsBucketName}.files`).findOne({ + _id: fileId, + 'metadata.ownerId': ownerId, + 'metadata.fencingToken': fence, + 'metadata.state': 'published' + }) + if (published !== null) throw error + await this.retireOwnedFile(fileId, ownerId) + } + await payloads.updateOne( + { _id: payloadId, state: 'uploading', ownerId, guard: ownerId }, + { + $set: { + state: 'deleted', + updatedAt: this.now(), + ...(fileId === undefined ? {} : { retiredFileId: fileId }) + }, + $unset: { inlineData: '', fileId: '' } + }, + { writeConcern: this.writeConcern } + ) + throw error + } + } + + /** Must be called in the same transaction that creates the referenced owner record. */ + public async addReference( + session: ClientSession, + reference: MongoPayloadReference, + operation: MongoPayloadOperationOptions = {} + ): Promise { + this.assertOperation(operation) + this.assertTransaction(session) + this.validateReference(reference) + const payloadId = this.payloadId(reference.scope, reference.payload) + const refs = this.db.collection(MongoCollectionNames.payloadReferences) + const payloads = this.db.collection(MongoCollectionNames.payloads) + const refId = mongoRecordKey( + mongoNodeKey(reference.scope), + reference.ownerKind, + reference.ownerId, + reference.slot + ) + const existing = await refs.findOne({ _id: refId }, { session, timeoutMS: operation.timeoutMS }) + if (existing !== null) { + if (existing.payloadId !== payloadId) + throw new Error('Mongo payload reference slot already names different content') + return + } + // This conditional write is the guard; a snapshot read followed by an insert is unsafe. + const guarded = await payloads.updateOne( + { _id: payloadId, state: 'ready' }, + { $set: { updatedAt: this.now() } }, + { session, timeoutMS: operation.timeoutMS } + ) + if (guarded.matchedCount !== 1) throw new Error('Mongo payload is not ready for reference') + await refs.insertOne( + { + _id: refId, + schemaVersion: 1, + network: reference.scope.network, + genesisHash: reference.scope.genesisHash, + nodeId: reference.scope.nodeId, + payloadId, + ownerKind: reference.ownerKind, + ownerId: reference.ownerId, + slot: reference.slot, + createdAt: this.now(), + updatedAt: this.now(), + ...(reference.expiresAt === undefined ? {} : { expiresAt: reference.expiresAt }) + }, + { session, timeoutMS: operation.timeoutMS } + ) + } + + /** Must be called in the transaction that releases the owner record's payload obligation. */ + public async releaseReference( + session: ClientSession, + reference: MongoPayloadReference, + operation: MongoPayloadOperationOptions = {} + ): Promise { + this.assertOperation(operation) + this.assertTransaction(session) + this.validateReference(reference) + const refId = mongoRecordKey( + mongoNodeKey(reference.scope), + reference.ownerKind, + reference.ownerId, + reference.slot + ) + const payloadId = this.payloadId(reference.scope, reference.payload) + const deleted = await this.db + .collection(MongoCollectionNames.payloadReferences) + .deleteOne({ _id: refId, payloadId }, { session, timeoutMS: operation.timeoutMS }) + if (deleted.deletedCount !== 1) throw new Error('Mongo payload reference does not exist') + } + + /** + * Creates one bounded manifest component and its durable pin in one caller + * transaction. S03 can combine this with its manifest owner record; this + * method does not parse or reconstruct BEEF. + */ + public async addManifestComponent( + session: ClientSession, + component: MongoManifestComponent + ): Promise { + this.assertTransaction(session) + if (!this.isUint64(component.ordinal) || component.manifestId.length === 0) + throw new Error('Invalid Mongo manifest component') + if (component.kind !== component.payload.kind) + throw new Error('Mongo manifest component kind mismatch') + const scope = this.scope + const ownerId = component.manifestId + const slot = component.ordinal + await this.addReference(session, { + scope, + payload: component.payload, + ownerKind: 'manifest', + ownerId, + slot + }) + const payloadId = this.payloadId(scope, component.payload) + const id = mongoRecordKey(mongoNodeKey(scope), component.manifestId, component.ordinal) + const existing = await this.db + .collection( + MongoCollectionNames.manifestComponents + ) + .findOne({ _id: id }, { session }) + if (existing !== null) { + if (existing.payloadId !== payloadId || existing.kind !== component.kind) + throw new Error('Mongo manifest ordinal already names different content') + return + } + const now = this.now() + await this.db + .collection(MongoCollectionNames.manifestComponents) + .insertOne( + { + _id: id, + schemaVersion: 1, + network: scope.network, + genesisHash: scope.genesisHash, + nodeId: scope.nodeId, + manifestId: component.manifestId, + ordinal: encodeMongoUint64(component.ordinal), + payloadId, + kind: component.kind, + createdAt: now, + updatedAt: now + }, + { session } + ) + } + + /** Claims logical deletion inside a caller transaction; physical deletion resumes outside it. */ + public async claimGarbage( + session: ClientSession, + payload: Pick, + operation: MongoPayloadOperationOptions = {} + ): Promise { + this.assertOperation(operation) + this.assertTransaction(session) + const payloadId = this.payloadId(this.scope, payload) + const refs = this.db.collection(MongoCollectionNames.payloadReferences) + const liveReferences = await refs.countDocuments( + { + payloadId, + $or: [{ ownerKind: { $ne: 'pin' } }, { $expr: { $gt: ['$expiresAt', '$$NOW'] } }] + }, + { session, timeoutMS: operation.timeoutMS } + ) + if (liveReferences !== 0) return false + const claimed = await this.db + .collection(MongoCollectionNames.payloads) + .findOneAndUpdate( + { _id: payloadId, state: 'ready' }, + [ + { + $set: { + state: 'deleting', + guard: randomUUID(), + updatedAt: '$$NOW', + leaseUntil: { + $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.uploadLeaseMs } + } + } + } + ], + { session, timeoutMS: operation.timeoutMS, returnDocument: 'after' } + ) + if (claimed === null) return false + await this.options.hooks?.afterDeleteClaim?.() + return true + } + + /** Safe to retry after process death. It only deletes the file named by the deleting record. */ + public async finishGarbage(payload: Pick): Promise { + const payloadId = this.payloadId(this.scope, payload) + const payloads = this.db.collection(MongoCollectionNames.payloads) + const record = await payloads.findOne({ _id: payloadId, state: 'deleting' }) + if (record === null) return false + if (record.fileId !== undefined) { + const owned = await this.db.collection(`${MongoGridFsBucketName}.files`).findOne({ + _id: record.fileId, + 'metadata.ownerId': record.ownerId, + 'metadata.fencingToken': record.fencingToken + }) + if (owned !== null) { + await this.bucket.delete(record.fileId).catch(error => { + if (!(error instanceof Error) || !/FileNotFound/.test(error.message)) throw error + }) + } + } + const result = await payloads.updateOne( + { _id: payloadId, state: 'deleting', guard: record.guard }, + { + $set: { + state: 'deleted', + updatedAt: this.now(), + ...(record.fileId === undefined ? {} : { retiredFileId: record.fileId }) + }, + $unset: { inlineData: '', fileId: '' } + }, + { writeConcern: this.writeConcern } + ) + return result.modifiedCount === 1 + } + + /** Recovery may retain content; it never guesses that a missing external owner is disposable. */ + public async recoverUploads(): Promise { + const payloads = this.db.collection(MongoCollectionNames.payloads) + const stale = await payloads + .find({ + network: this.scope.network, + genesisHash: this.scope.genesisHash, + state: 'uploading', + $expr: { $lte: ['$leaseUntil', '$$NOW'] } + }) + .toArray() + for (const record of stale) { + if (record.fileId !== undefined) { + try { + await this.verifyGridFs(record.fileId, { + digest: record.digest, + byteLength: BigInt(record.byteLength).toString() + }) + await this.db.collection(`${MongoGridFsBucketName}.files`).updateOne( + { + _id: record.fileId, + 'metadata.ownerId': record.ownerId, + 'metadata.fencingToken': record.fencingToken + }, + { $set: { 'metadata.state': 'published' } }, + { writeConcern: this.writeConcern } + ) + const recovered = await payloads.updateOne( + { _id: record._id, state: 'uploading', ownerId: record.ownerId, guard: record.guard }, + { $set: { state: 'ready', updatedAt: this.now() }, $unset: { uploadId: '' } }, + { writeConcern: this.writeConcern } + ) + if (recovered.modifiedCount === 1) continue + } catch { + /* an incomplete upload is retired below */ + } + } + const current = await payloads.findOne({ _id: record._id }) + if ( + current === null || + current.state !== 'uploading' || + current.ownerId !== record.ownerId || + current.guard !== record.guard + ) { + continue + } + if (record.fileId !== undefined) + await this.retireOwnedFile(record.fileId, record.ownerId, record.fencingToken) + await payloads.updateOne( + { _id: record._id, state: 'uploading', ownerId: record.ownerId, guard: record.guard }, + { + $set: { + state: 'deleted', + updatedAt: this.now(), + ...(record.fileId === undefined ? {} : { retiredFileId: record.fileId }) + }, + $unset: { inlineData: '', fileId: '' } + }, + { writeConcern: this.writeConcern } + ) + } + const deleting = await payloads + .find({ network: this.scope.network, genesisHash: this.scope.genesisHash, state: 'deleting' }) + .toArray() + for (const record of deleting) + await this.finishGarbage({ kind: record.kind, digest: record.digest }) + } + + private async reserveUpload( + payloads: Collection, + payloadId: string, + input: MongoPayloadInput + ): Promise<{ ready: true; record: PayloadDocument } | { ready: false; record: PayloadDocument }> { + const deadline = Date.now() + this.uploadLeaseMs + while (true) { + this.throwIfAborted(input.signal) + const observed = await payloads.findOne({ _id: payloadId }) + if (observed?.state === 'ready') return { ready: true, record: observed } + const ownerId = randomUUID() + const fence = encodeMongoUint64( + (BigInt(observed?.fencingToken ?? '00000000000000000000') + BigInt(1)).toString() + ) + try { + const claimed = await payloads.findOneAndUpdate( + observed === null + ? { _id: payloadId, state: { $exists: false } } + : { + _id: payloadId, + state: observed.state, + guard: observed.guard, + fencingToken: observed.fencingToken, + $or: [{ state: 'deleted' }, { $expr: { $lte: ['$leaseUntil', '$$NOW'] } }] + }, + [ + { + $set: { + schemaVersion: { $ifNull: ['$schemaVersion', 1] }, + network: { $ifNull: ['$network', this.scope.network] }, + genesisHash: { $ifNull: ['$genesisHash', this.scope.genesisHash] }, + kind: { $ifNull: ['$kind', input.kind] }, + digest: { $ifNull: ['$digest', input.digest] }, + byteLength: { $ifNull: ['$byteLength', encodeMongoUint64(input.byteLength)] }, + createdAt: { $ifNull: ['$createdAt', '$$NOW'] }, + state: 'uploading', + guard: ownerId, + ownerNodeId: this.scope.nodeId, + ownerId, + fencingToken: fence, + leaseUntil: { + $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.uploadLeaseMs } + }, + updatedAt: '$$NOW', + uploadId: ownerId + } + }, + { $unset: ['inlineData', 'fileId'] } + ], + { upsert: observed === null, returnDocument: 'after', writeConcern: this.writeConcern } + ) + if (claimed !== null && claimed.ownerId === ownerId && claimed.guard === ownerId) { + if (observed?.fileId !== undefined) + await this.retireOwnedFile(observed.fileId, observed.ownerId, observed.fencingToken) + return { ready: false, record: claimed } + } + } catch (error) { + if (!duplicateKey(error)) throw error + } + const ready = await payloads.findOne({ _id: payloadId, state: 'ready' }) + if (ready !== null) return { ready: true, record: ready } + if (Date.now() > deadline) throw new Error('Mongo payload upload reservation was lost') + await delay(50) + } + } + + private async streamInput( + input: MongoPayloadInput, + payloadId: string, + ownerId: string, + fencingToken: string + ): Promise<{ + digest: string + length: bigint + txid?: string + inline?: Uint8Array[] + fileId?: ObjectId + }> { + const hash = createHash('sha256') + let length = BigInt(0) + let inline: Uint8Array[] | undefined = [] + let upload: ReturnType | undefined + let fileId: ObjectId | undefined + const iterator = input.bytes[Symbol.asyncIterator]() + try { + while (true) { + const next = await this.nextWithAbort(iterator, input.signal) + if (next.done) break + const chunk = next.value + if (input.signal?.aborted) + throw input.signal.reason ?? new Error('Mongo payload upload aborted') + if (!(chunk instanceof Uint8Array)) + throw new Error('Mongo payload stream yielded a non-byte chunk') + length += BigInt(chunk.byteLength) + if (length > BigInt(input.byteLength)) + throw new Error('Mongo payload stream exceeds declared length') + if (length > this.maxPayloadBytes) + throw new Error('Mongo payload stream exceeds configured byte bound') + hash.update(chunk) + if (inline !== undefined) { + inline.push(chunk) + if (length > BigInt(this.inlineCeiling)) { + fileId = new ObjectId() + upload = this.bucket.openUploadStreamWithId(fileId, payloadId, { + chunkSizeBytes: this.gridFsChunkBytes, + metadata: { + state: 'staged', + payloadId, + kind: input.kind, + digest: input.digest, + byteLength: encodeMongoUint64(input.byteLength), + ownerId, + fencingToken + } + }) + for (const buffered of inline) + if (!upload.write(Buffer.from(buffered))) + await this.awaitAbort(once(upload, 'drain'), input.signal) + inline = undefined + } + } else if (upload === undefined) throw new Error('Mongo payload stream writer missing') + else if (!upload.write(Buffer.from(chunk))) + await this.awaitAbort(once(upload, 'drain'), input.signal) + } + if (upload !== undefined) { + upload.end() + await this.awaitAbort(once(upload, 'finish'), input.signal) + } + } catch (error) { + upload?.destroy(error instanceof Error ? error : new Error('Mongo payload stream failed')) + await iterator.return?.().catch(() => undefined) + throw error + } + const digest = hash.digest() + const result: { + digest: string + length: bigint + txid?: string + inline?: Uint8Array[] + fileId?: ObjectId + } = { digest: digest.toString('hex'), length, inline, fileId } + if (input.kind === 'raw-transaction') + result.txid = createHash('sha256').update(digest).digest().reverse().toString('hex') + return result + } + + private fitsInline( + payloadId: string, + input: MongoPayloadInput, + data: Binary, + now: Date + ): boolean { + const candidate = { + _id: payloadId, + schemaVersion: 1, + network: this.scope.network, + genesisHash: this.scope.genesisHash, + kind: input.kind, + digest: input.digest, + byteLength: encodeMongoUint64(input.byteLength), + state: 'ready', + guard: '0'.repeat(36), + ownerNodeId: this.scope.nodeId, + ownerId: '0'.repeat(36), + fencingToken: encodeMongoUint64('1'), + leaseUntil: now, + createdAt: now, + updatedAt: now, + inlineData: data + } + return BSON.serialize(candidate).byteLength < 1024 * 1024 + } + + private async retireOwnedFile( + fileId: ObjectId, + ownerId: string, + fencingToken?: string + ): Promise { + const file = await this.db.collection(`${MongoGridFsBucketName}.files`).findOne({ + _id: fileId, + 'metadata.ownerId': ownerId, + ...(fencingToken === undefined ? {} : { 'metadata.fencingToken': fencingToken }) + }) + if (file !== null) await this.bucket.delete(fileId).catch(() => undefined) + } + + private async verifyGridFs( + fileId: ObjectId, + input: Pick, + signal?: AbortSignal + ): Promise { + const hash = createHash('sha256') + let length = BigInt(0) + const download = this.bucket.openDownloadStream(fileId) + const iterator = download[Symbol.asyncIterator]() + try { + while (true) { + const next = await this.nextWithAbort(iterator, signal) + if (next.done) break + const chunk = next.value + if (!(chunk instanceof Uint8Array)) + throw new Error('Mongo GridFS returned a non-byte chunk') + length += BigInt(chunk.byteLength) + hash.update(chunk) + } + } catch (error) { + download.destroy( + error instanceof Error ? error : new Error('Mongo GridFS verification failed') + ) + throw error + } + if (length !== BigInt(input.byteLength) || hash.digest('hex') !== input.digest) + throw new Error('Mongo GridFS staged payload verification failed') + } + + private refFor(input: MongoPayloadInput, record?: PayloadDocument): MongoPayloadRef { + if (record !== undefined && decodeMongoUint64(record.byteLength) !== input.byteLength) + throw new Error('Mongo payload byte length does not match ready content') + const ref: MongoPayloadRef = { + kind: input.kind, + digest: input.digest, + byteLength: input.byteLength + } + if (input.kind === 'raw-transaction') { + ref.txid = createHash('sha256') + .update(Buffer.from(input.digest, 'hex')) + .digest() + .reverse() + .toString('hex') + if (input.txid !== undefined && input.txid !== ref.txid) + throw new Error('Mongo raw transaction id does not match bytes') + } + return ref + } + + private payloadId( + scope: MongoChainScope, + payload: Pick + ): string { + if (!HEX_256.test(payload.digest)) throw new Error('Invalid Mongo payload digest') + return mongoRecordKey(mongoChainKey(scope), payload.kind, payload.digest) + } + + private validateInput(input: MongoPayloadInput): void { + this.payloadId(this.scope, input) + if (!this.isUint64(input.byteLength)) throw new Error('Invalid Mongo payload byte length') + if (BigInt(input.byteLength) > this.maxPayloadBytes) + throw new Error('Mongo payload exceeds configured byte bound') + if (input.txid !== undefined && (input.kind !== 'raw-transaction' || !HEX_256.test(input.txid))) + throw new Error('Invalid Mongo raw transaction id') + } + + private validateReference(reference: MongoPayloadReference): void { + this.payloadId(reference.scope, reference.payload) + if ( + reference.scope.network !== this.scope.network || + reference.scope.genesisHash !== this.scope.genesisHash || + reference.scope.nodeId !== this.scope.nodeId + ) + throw new Error('Mongo payload reference scope does not match this store') + if (reference.ownerId.length === 0 || reference.slot.length === 0) + throw new Error('Invalid Mongo payload reference owner') + if ((reference.ownerKind === 'pin') !== (reference.expiresAt !== undefined)) + throw new Error('Mongo payload expiry is valid only for explicit pins') + if (reference.expiresAt !== undefined && reference.expiresAt <= this.now()) + throw new Error('Mongo payload pin must be unexpired') + } + + private assertTransaction(session: ClientSession): void { + if (!session.inTransaction()) + throw new Error('Mongo payload reference changes require an active transaction') + } + + private assertOperation(operation: MongoPayloadOperationOptions): void { + if (operation.signal?.aborted) + throw operation.signal.reason ?? new Error('Mongo payload operation aborted') + if ( + operation.timeoutMS !== undefined && + (!Number.isSafeInteger(operation.timeoutMS) || operation.timeoutMS < 1) + ) + throw new Error('Invalid Mongo payload operation timeout') + } + + private async nextWithAbort( + iterator: AsyncIterator, + signal?: AbortSignal + ): Promise> { + if (signal === undefined) return await iterator.next() + if (signal.aborted) throw signal.reason ?? new Error('Mongo payload upload aborted') + return await this.raceAbort(iterator.next(), signal) + } + + private async awaitAbort(promise: Promise, signal?: AbortSignal): Promise { + if (signal === undefined) return await promise + if (signal.aborted) throw signal.reason ?? new Error('Mongo payload upload aborted') + return await this.raceAbort(promise, signal) + } + + private async raceAbort(promise: Promise, signal: AbortSignal): Promise { + return await new Promise((resolve, reject) => { + const abort = () => { + cleanup() + reject(signal.reason ?? new Error('Mongo payload upload aborted')) + } + const cleanup = () => signal.removeEventListener('abort', abort) + signal.addEventListener('abort', abort, { once: true }) + promise.then( + value => { + cleanup() + resolve(value) + }, + error => { + cleanup() + reject(error) + } + ) + }) + } + + private throwIfAborted(signal?: AbortSignal): void { + if (signal?.aborted) throw signal.reason ?? new Error('Mongo payload upload aborted') + } + + private isUint64(value: string): boolean { + return /^(0|[1-9][0-9]{0,19})$/.test(value) && BigInt(value) <= MAX_UINT64 + } +} diff --git a/packages/overlays/overlay/src/storage/mongo/MongoReadGuards.ts b/packages/overlays/overlay/src/storage/mongo/MongoReadGuards.ts new file mode 100644 index 000000000..2b6f40d39 --- /dev/null +++ b/packages/overlays/overlay/src/storage/mongo/MongoReadGuards.ts @@ -0,0 +1,209 @@ +import { randomUUID } from 'node:crypto' +import type { ClientSession, Db, Document } from 'mongodb' +import type { StorageScope } from '../AdmissionStorage.js' +import { MongoCollectionNames, mongoNodeKey, mongoRecordKey } from './MongoSchema.js' + +const maxTimeoutMs = 30_000 + +export interface MongoReadGuard { + scope: StorageScope + /** A bounded logical-record identifier, never a dynamic BSON key. */ + key: string + /** `null` is an actual absent-record predicate, represented by a sentinel row. */ + expectedVersion: string | null +} + +export interface MongoReadGuardOperationOptions { + timeoutMS?: number + signal?: AbortSignal +} + +interface ReadGuardDocument extends Document { + _id: string + schemaVersion: 1 + network: string + genesisHash: string + nodeId: string + key: string + version: string | null + guard: string + createdAt: Date + updatedAt: Date +} + +/** A semantic predicate conflict, as distinct from a transient Mongo transaction retry. */ +export class MongoReadGuardConflictError extends Error { + public readonly code = 'mongo-read-guard-conflict' + + public constructor(read: MongoReadGuard) { + super(`Mongo read guard conflict for ${read.key}`) + this.name = 'MongoReadGuardConflictError' + } +} + +/** + * Durable optimistic read predicates for the Mongo admission transaction. + * Call `initialize` before starting a transaction; `check` and `changeVersion` + * intentionally require an active transaction and always write the sentinel. + */ +export class MongoReadGuards { + public constructor(private readonly db: Db) {} + + /** + * Creates a version-null sentinel with majority+journal durability. This must + * run before the caller opens the admission transaction, so a negative read + * has a real row to conflict with a concurrent writer. + */ + public async initialize( + scope: StorageScope, + key: string, + operation: MongoReadGuardOperationOptions = {} + ): Promise { + this.assertOperation(operation) + const id = this.idFor(scope, key) + const now = new Date() + const collection = this.collection() + try { + await collection.updateOne( + { _id: id }, + { + $setOnInsert: { + _id: id, + schemaVersion: 1, + network: scope.network, + genesisHash: scope.genesisHash, + nodeId: scope.nodeId, + key, + version: null, + guard: randomUUID(), + createdAt: now, + updatedAt: now + } + }, + { + upsert: true, + timeoutMS: operation.timeoutMS, + writeConcern: { w: 'majority', j: true } + } + ) + } catch (error) { + if ((error as { code?: number }).code !== 11000) throw error + } + this.assertOperation(operation) + const sentinel = await collection.findOne( + { _id: id }, + { + timeoutMS: operation.timeoutMS, + readConcern: { level: 'majority' } + } + ) + if ( + sentinel === null || + sentinel.network !== scope.network || + sentinel.genesisHash !== scope.genesisHash || + sentinel.nodeId !== scope.nodeId || + sentinel.key !== key + ) { + throw new Error('Incompatible Mongo read guard sentinel') + } + } + + /** Checks a persisted version predicate by updating its random guard inside the transaction. */ + public async check( + session: ClientSession, + read: MongoReadGuard, + operation: MongoReadGuardOperationOptions = {} + ): Promise { + this.assertTransaction(session) + this.assertOperation(operation) + const id = this.idFor(read.scope, read.key) + this.validateVersion(read.expectedVersion) + const result = await this.collection().updateOne( + { _id: id, version: read.expectedVersion }, + { $set: { guard: randomUUID(), updatedAt: new Date() } }, + { session, timeoutMS: operation.timeoutMS } + ) + if (result.matchedCount !== 1) throw new MongoReadGuardConflictError(read) + this.assertOperation(operation) + } + + /** Changes version and guard in one transactionally fenced row write. */ + public async changeVersion( + session: ClientSession, + read: MongoReadGuard, + nextVersion: string | null, + operation: MongoReadGuardOperationOptions = {} + ): Promise { + this.assertTransaction(session) + this.assertOperation(operation) + const id = this.idFor(read.scope, read.key) + this.validateVersion(read.expectedVersion) + this.validateVersion(nextVersion) + const result = await this.collection().updateOne( + { _id: id, version: read.expectedVersion }, + { $set: { version: nextVersion, guard: randomUUID(), updatedAt: new Date() } }, + { session, timeoutMS: operation.timeoutMS } + ) + if (result.matchedCount !== 1) throw new MongoReadGuardConflictError(read) + this.assertOperation(operation) + } + + private collection() { + return this.db.collection(MongoCollectionNames.readGuards) + } + + private idFor(scope: StorageScope, key: string): string { + mongoNodeKey(scope) + this.validatePart(key, 1024, 'key') + return mongoRecordKey(mongoNodeKey(scope), 'read-guard', key) + } + + private validateVersion(version: string | null): void { + if (version === null) return + this.validatePart(version, 256, 'version') + } + + private validatePart(value: string, maximumBytes: number, label: string): void { + if ( + typeof value !== 'string' || + value.length === 0 || + Buffer.byteLength(value, 'utf8') > maximumBytes || + value.includes('\u0000') || + !this.isWellFormed(value) + ) { + throw new Error(`Invalid Mongo read guard ${label}`) + } + } + + private isWellFormed(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const unit = value.charCodeAt(index) + if (unit >= 0xd800 && unit <= 0xdbff) { + if (index + 1 >= value.length) return false + const next = value.charCodeAt(index + 1) + if (next < 0xdc00 || next > 0xdfff) return false + index += 1 + } else if (unit >= 0xdc00 && unit <= 0xdfff) { + return false + } + } + return true + } + + private assertTransaction(session: ClientSession): void { + if (!session.inTransaction()) throw new Error('Mongo read guards require an active transaction') + } + + private assertOperation(operation: MongoReadGuardOperationOptions): void { + if (operation.signal?.aborted) + throw operation.signal.reason ?? new Error('Mongo read guard operation aborted') + if ( + operation.timeoutMS !== undefined && + (!Number.isSafeInteger(operation.timeoutMS) || + operation.timeoutMS < 1 || + operation.timeoutMS > maxTimeoutMs) + ) { + throw new Error('Invalid Mongo read guard operation timeout') + } + } +} diff --git a/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts b/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts new file mode 100644 index 000000000..91cd7d965 --- /dev/null +++ b/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts @@ -0,0 +1,964 @@ +import { createHash } from 'node:crypto' +import type { CollectionInfo, Db, Document, IndexDescription } from 'mongodb' +import { + parseStorageOutputIndex, + parseStorageUint64, + type StorageScope +} from '../AdmissionStorage.js' + +/** The immutable chain namespace shared by all Overlay nodes. */ +export interface MongoChainScope { + network: string + genesisHash: string +} + +export const MongoCollectionNames = { + schema: 'overlay_schema', + payloads: 'overlay_payloads', + payloadReferences: 'overlay_payload_references', + submissionOperations: 'overlay_submission_operations', + readGuards: 'overlay_read_guards', + transactions: 'overlay_transactions', + transactionProofs: 'overlay_transaction_proofs', + outputs: 'overlay_outputs', + consumptionEdges: 'overlay_consumption_edges', + appliedTransactions: 'overlay_applied_transactions', + topicGenerations: 'overlay_topic_generations', + topicAnchors: 'overlay_topic_anchors', + topicAnchorTips: 'overlay_topic_anchor_tips', + basmRecoveryJobs: 'overlay_basm_recovery_jobs', + basmComparisons: 'overlay_basm_comparisons', + gaspCursors: 'overlay_gasp_cursors', + gaspGraphs: 'overlay_gasp_graphs', + gaspNodes: 'overlay_gasp_nodes', + manifestComponents: 'overlay_manifest_components', + lookupOutbox: 'overlay_lookup_outbox', + propagationOutbox: 'overlay_propagation_outbox', + shipRecords: 'overlay_ship_records', + slapRecords: 'overlay_slap_records', + bannedRecords: 'overlay_banned_records' +} as const + +/** GridFS bucket for payload bytes that cannot be stored inline. */ +export const MongoGridFsBucketName = 'overlayPayloads' + +const schemaVersion = 1 +const maxUint64 = '18446744073709551615' +const paddedUint64Pattern = '^[0-9]{20}$' +const hashPattern = '^[0-9a-f]{64}$' +const uuidPattern = '^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$' +const safeStringPattern = '^[^\\x00]+$' + +function isWellFormed(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const unit = value.charCodeAt(index) + if (unit >= 0xd800 && unit <= 0xdbff) { + if (index + 1 >= value.length) return false + const next = value.charCodeAt(index + 1) + if (next < 0xdc00 || next > 0xdfff) return false + index += 1 + } else if (unit >= 0xdc00 && unit <= 0xdfff) { + return false + } + } + return true +} + +function validPart(value: string): string { + if ( + typeof value !== 'string' || + value.length === 0 || + Buffer.byteLength(value, 'utf8') > 1024 || + value.includes('\u0000') || + !isWellFormed(value) + ) { + throw new Error('Invalid Mongo record key component') + } + return value +} + +/** + * A length-framed, UTF-8 tuple. It remains unambiguous when values contain + * dots, colons, or separators and is safe to use only as a BSON value, never + * as a dynamic BSON field name. + */ +export function mongoRecordKey(...parts: string[]): string { + if (parts.length === 0 || parts.length > 32) throw new Error('Invalid Mongo record key') + const encoded = parts.map(part => { + const valid = validPart(part) + return `${Buffer.byteLength(valid, 'utf8')}:${valid}` + }) + const result = `v1|${encoded.join('|')}` + if (Buffer.byteLength(result, 'utf8') > 4096) throw new Error('Mongo record key is too large') + return result +} + +function validChainScope(scope: MongoChainScope): MongoChainScope { + if ( + typeof scope !== 'object' || + scope === null || + typeof scope.network !== 'string' || + typeof scope.genesisHash !== 'string' || + Buffer.byteLength(validPart(scope.network), 'utf8') > 128 || + !new RegExp(hashPattern).test(scope.genesisHash) + ) { + throw new Error('Invalid Mongo chain scope') + } + return scope +} + +export function mongoChainKey(scope: MongoChainScope): string { + const valid = validChainScope(scope) + return mongoRecordKey('chain', valid.network, valid.genesisHash) +} + +export function mongoNodeKey(scope: StorageScope): string { + const valid = validChainScope(scope) + return mongoRecordKey('node', valid.network, valid.genesisHash, validPart(scope.nodeId)) +} + +/** Stores a canonical S01 uint64 in an exact, lexically sortable BSON string. */ +export function encodeMongoUint64(value: string): string { + const parsed = parseStorageUint64(value) + return parsed.toString(10).padStart(20, '0') +} + +/** Strict inverse of encodeMongoUint64; rejects non-canonical padded values. */ +export function decodeMongoUint64(value: string): string { + if (typeof value !== 'string' || !new RegExp(paddedUint64Pattern).test(value)) + throw new Error('Invalid Mongo uint64') + if (value > maxUint64) throw new Error('Invalid Mongo uint64') + const decoded = BigInt(value).toString(10) + if (encodeMongoUint64(decoded) !== value) throw new Error('Invalid Mongo uint64') + return decoded +} + +/** Validates the wire uint32 domain and returns its canonical decimal spelling. */ +export function encodeMongoOutputIndex(value: string): string { + return String(parseStorageOutputIndex(value)) +} + +const string = (maxLength = 1024): Document => ({ + bsonType: 'string', + pattern: safeStringPattern, + minLength: 1, + maxLength +}) +const date: Document = { bsonType: 'date' } +const hash: Document = { bsonType: 'string', pattern: hashPattern } +const uint64: Document = { bsonType: 'string', pattern: paddedUint64Pattern } +const uint32: Document = { bsonType: 'string', pattern: '^(0|[1-9][0-9]{0,9})$' } +const schemaV1: Document = { bsonType: 'int', enum: [schemaVersion] } + +const chainFields: Document = { network: string(128), genesisHash: hash } +const nodeFields: Document = { ...chainFields, nodeId: string(256) } +const commonFields: Document = { + schemaVersion: schemaV1, + createdAt: date, + updatedAt: date +} + +function scopedValidator( + fields: Document, + required: string[], + options: { node?: boolean; extra?: Document; allowAdditional?: boolean } = {} +): Document { + const scopeFields = options.node ? nodeFields : chainFields + const uint64Bounds = Object.entries(fields) + .filter(([, definition]) => definition === uint64) + .map(([field]) => ({ + $or: [{ $eq: [{ $type: `$${field}` }, 'missing'] }, { $lte: [`$${field}`, maxUint64] }] + })) + return { + $and: [ + { + $jsonSchema: { + bsonType: 'object', + additionalProperties: options.allowAdditional ?? false, + required: [ + '_id', + 'schemaVersion', + 'createdAt', + 'updatedAt', + ...Object.keys(scopeFields), + ...required + ], + properties: { _id: string(4096), ...commonFields, ...scopeFields, ...fields } + } + }, + ...uint64Bounds.map(bound => ({ $expr: bound })), + ...(options.extra === undefined ? [] : [options.extra]) + ] + } +} + +export interface MongoCollectionDefinition { + readonly name: (typeof MongoCollectionNames)[keyof typeof MongoCollectionNames] + readonly validator: Document + readonly indexes: readonly IndexDescription[] +} + +interface MongoSchemaDocument extends Document { + _id: string +} + +const index = ( + key: Document, + name: string, + options: Omit = {} +): IndexDescription => ({ + key, + name, + ...options +}) + +const definitions: MongoCollectionDefinition[] = [ + { + name: MongoCollectionNames.schema, + validator: scopedValidator( + { schemaFingerprint: hash, lastTransactionProbeAt: date }, + ['schemaFingerprint'], + { node: true } + ), + indexes: [index({ network: 1, genesisHash: 1, nodeId: 1 }, 'scope_unique', { unique: true })] + }, + { + name: MongoCollectionNames.payloads, + validator: scopedValidator( + { + kind: string(64), + digest: hash, + byteLength: uint64, + state: { bsonType: 'string', enum: ['uploading', 'ready', 'deleting', 'deleted'] }, + guard: string(256), + ownerNodeId: string(256), + ownerId: string(1024), + fencingToken: uint64, + leaseUntil: date, + fileId: { bsonType: 'objectId' }, + inlineData: { bsonType: 'binData' }, + retiredFileId: { bsonType: 'objectId' }, + uploadId: string(256) + }, + [ + 'kind', + 'digest', + 'byteLength', + 'state', + 'guard', + 'ownerNodeId', + 'ownerId', + 'fencingToken', + 'leaseUntil' + ] + ), + indexes: [ + index({ network: 1, genesisHash: 1, kind: 1, digest: 1 }, 'chain_kind_digest_unique', { + unique: true + }), + index({ network: 1, genesisHash: 1, state: 1, updatedAt: 1 }, 'chain_state_updated') + ] + }, + { + name: MongoCollectionNames.payloadReferences, + validator: scopedValidator( + { + payloadId: string(4096), + ownerKind: { + bsonType: 'string', + enum: [ + 'transaction', + 'applied-history', + 'output', + 'gasp-graph', + 'gasp-node', + 'basm-job', + 'lookup-outbox', + 'propagation-outbox', + 'manifest', + 'pin' + ] + }, + ownerId: string(4096), + slot: string(256), + expiresAt: date + }, + ['payloadId', 'ownerKind', 'ownerId', 'slot'], + { + node: true, + extra: { + $expr: { + $eq: [{ $eq: ['$ownerKind', 'pin'] }, { $ne: [{ $type: '$expiresAt' }, 'missing'] }] + } + } + } + ), + indexes: [ + index( + { network: 1, genesisHash: 1, nodeId: 1, ownerKind: 1, ownerId: 1, slot: 1 }, + 'owner_slot_unique', + { + unique: true + } + ), + index({ payloadId: 1 }, 'payload_reference'), + index({ expiresAt: 1 }, 'pin_expiry', { partialFilterExpression: { ownerKind: 'pin' } }) + ] + }, + { + name: MongoCollectionNames.submissionOperations, + validator: scopedValidator( + { + operationId: string(1024), + semanticDigest: hash, + txid: hash, + state: { bsonType: 'string', enum: ['pending', 'committed', 'aborted'] }, + attemptId: { bsonType: 'string', pattern: uuidPattern }, + leaseOwner: { bsonType: 'string', pattern: uuidPattern }, + leaseToken: uint64, + leaseUntil: date, + guard: { bsonType: 'string', pattern: uuidPattern }, + receipt: { bsonType: 'binData' } + }, + [ + 'operationId', + 'semanticDigest', + 'txid', + 'state', + 'attemptId', + 'leaseOwner', + 'leaseToken', + 'leaseUntil', + 'guard' + ], + { + node: true, + extra: { + $expr: { + $cond: [ + { $eq: [{ $type: '$receipt' }, 'missing'] }, + true, + { $lte: [{ $binarySize: '$receipt' }, 1048576] } + ] + } + } + } + ), + indexes: [ + index({ network: 1, genesisHash: 1, nodeId: 1, operationId: 1 }, 'operation_unique', { + unique: true + }), + index({ network: 1, genesisHash: 1, nodeId: 1, txid: 1, state: 1 }, 'scope_tx_state') + ] + }, + { + name: MongoCollectionNames.readGuards, + validator: scopedValidator( + { + key: string(4096), + version: { bsonType: ['string', 'null'], maxLength: 256 }, + guard: { bsonType: 'string', pattern: uuidPattern } + }, + ['key', 'version', 'guard'], + { node: true } + ), + indexes: [ + index({ network: 1, genesisHash: 1, nodeId: 1, key: 1, guard: 1 }, 'read_guard_unique', { + unique: true + }), + index({ guard: 1 }, 'guard_lookup') + ] + }, + { + name: MongoCollectionNames.transactions, + validator: scopedValidator( + { + txid: hash, + rawPayloadId: string(4096), + manifestPayloadId: string(4096), + blockHash: hash, + blockHeight: uint64 + }, + ['txid'] + ), + indexes: [ + index({ network: 1, genesisHash: 1, txid: 1 }, 'chain_txid_unique', { unique: true }), + index({ network: 1, genesisHash: 1, blockHash: 1, blockHeight: 1 }, 'chain_block') + ] + }, + { + name: MongoCollectionNames.transactionProofs, + validator: scopedValidator( + { + txid: hash, + proofPayloadId: string(4096), + blockHash: hash, + blockHeight: uint64, + variantDigest: hash + }, + ['txid', 'proofPayloadId', 'variantDigest'] + ), + indexes: [ + index( + { network: 1, genesisHash: 1, txid: 1, variantDigest: 1 }, + 'transaction_variant_unique', + { unique: true } + ) + ] + }, + { + name: MongoCollectionNames.outputs, + validator: scopedValidator( + { + topic: string(256), + txid: hash, + outputIndex: uint32, + satoshis: uint64, + score: uint64, + scriptPayloadId: string(4096), + scriptOffset: uint64, + scriptByteLength: uint64, + state: { bsonType: 'string', enum: ['unspent', 'spent', 'evicted'] }, + spender: string(4096), + version: string(256) + }, + [ + 'topic', + 'txid', + 'outputIndex', + 'satoshis', + 'score', + 'scriptPayloadId', + 'scriptOffset', + 'scriptByteLength', + 'state', + 'version' + ], + { node: true, extra: { $expr: { $lte: ['$outputIndex', '4294967295'] } } } + ), + indexes: [ + index( + { network: 1, genesisHash: 1, nodeId: 1, topic: 1, txid: 1, outputIndex: 1 }, + 'outpoint_unique', + { unique: true } + ), + index( + { network: 1, genesisHash: 1, nodeId: 1, topic: 1, state: 1, score: 1, _id: 1 }, + 'gasp_order' + ) + ] + }, + { + name: MongoCollectionNames.consumptionEdges, + validator: scopedValidator( + { + topic: string(256), + sourceTxid: hash, + sourceOutputIndex: uint32, + consumerTxid: hash, + consumerOutputIndex: uint32 + }, + ['topic', 'sourceTxid', 'sourceOutputIndex', 'consumerTxid', 'consumerOutputIndex'], + { + node: true, + extra: { + $expr: { + $and: [ + { $lte: ['$sourceOutputIndex', '4294967295'] }, + { $lte: ['$consumerOutputIndex', '4294967295'] } + ] + } + } + } + ), + indexes: [ + index( + { + network: 1, + genesisHash: 1, + nodeId: 1, + topic: 1, + sourceTxid: 1, + sourceOutputIndex: 1, + consumerTxid: 1, + consumerOutputIndex: 1 + }, + 'edge_unique', + { unique: true } + ), + index( + { network: 1, genesisHash: 1, nodeId: 1, consumerTxid: 1, consumerOutputIndex: 1 }, + 'consumer_lookup' + ) + ] + }, + { + name: MongoCollectionNames.appliedTransactions, + validator: scopedValidator( + { + topic: string(256), + txid: hash, + state: { bsonType: 'string', enum: ['active', 'unproven', 'evicted'] }, + firstSeenHeight: uint64, + proofPayloadId: string(4096), + admissionId: string(4096) + }, + ['topic', 'txid', 'state', 'admissionId'], + { node: true } + ), + indexes: [ + index({ network: 1, genesisHash: 1, nodeId: 1, topic: 1, txid: 1 }, 'topic_txid_unique', { + unique: true + }), + index( + { network: 1, genesisHash: 1, nodeId: 1, topic: 1, state: 1, firstSeenHeight: 1 }, + 'topic_state_height' + ) + ] + } +] + +function stateDefinition( + name: MongoCollectionDefinition['name'], + fields: Document, + required: string[], + node: boolean, + indexes: readonly IndexDescription[], + extra?: Document +): void { + definitions.push({ name, validator: scopedValidator(fields, required, { node, extra }), indexes }) +} + +stateDefinition( + MongoCollectionNames.topicGenerations, + { topic: string(256), chainEpoch: uint64, topicHistoryGeneration: uint64, policyId: string(256) }, + ['topic', 'chainEpoch', 'topicHistoryGeneration', 'policyId'], + true, + [index({ network: 1, genesisHash: 1, nodeId: 1, topic: 1 }, 'topic_unique', { unique: true })] +) +stateDefinition( + MongoCollectionNames.topicAnchors, + { + topic: string(256), + chainEpoch: uint64, + topicHistoryGeneration: uint64, + height: uint64, + blockHash: hash, + blockIndex: uint64, + basmRoot: hash, + tac: hash, + previousTac: hash, + state: { bsonType: 'string', enum: ['canonical', 'noncanonical'] } + }, + [ + 'topic', + 'chainEpoch', + 'topicHistoryGeneration', + 'height', + 'blockHash', + 'blockIndex', + 'basmRoot', + 'tac', + 'previousTac', + 'state' + ], + true, + [ + index( + { + network: 1, + genesisHash: 1, + nodeId: 1, + topic: 1, + chainEpoch: 1, + topicHistoryGeneration: 1, + height: 1, + blockHash: 1 + }, + 'anchor_unique', + { unique: true } + ) + ] +) +stateDefinition( + MongoCollectionNames.topicAnchorTips, + { + topic: string(256), + chainEpoch: uint64, + topicHistoryGeneration: uint64, + height: uint64, + blockHash: hash, + anchorId: string(4096) + }, + ['topic', 'chainEpoch', 'topicHistoryGeneration', 'height', 'blockHash', 'anchorId'], + true, + [index({ network: 1, genesisHash: 1, nodeId: 1, topic: 1 }, 'tip_unique', { unique: true })] +) +stateDefinition( + MongoCollectionNames.basmRecoveryJobs, + { + topic: string(256), + peerId: string(1024), + jobId: string(1024), + chainEpoch: uint64, + topicHistoryGeneration: uint64, + leaseToken: uint64, + leaseUntil: date, + state: string(64), + checkpoint: string(4096) + }, + [ + 'topic', + 'peerId', + 'jobId', + 'chainEpoch', + 'topicHistoryGeneration', + 'leaseToken', + 'leaseUntil', + 'state', + 'checkpoint' + ], + true, + [ + index({ network: 1, genesisHash: 1, nodeId: 1, topic: 1, peerId: 1, jobId: 1 }, 'job_unique', { + unique: true + }), + index({ state: 1, leaseUntil: 1 }, 'lease_state') + ] +) +stateDefinition( + MongoCollectionNames.basmComparisons, + { + topic: string(256), + peerId: string(1024), + chainEpoch: uint64, + topicHistoryGeneration: uint64, + commonHeight: uint64, + commonHash: hash, + state: string(64) + }, + [ + 'topic', + 'peerId', + 'chainEpoch', + 'topicHistoryGeneration', + 'commonHeight', + 'commonHash', + 'state' + ], + true, + [ + index( + { + network: 1, + genesisHash: 1, + nodeId: 1, + topic: 1, + chainEpoch: 1, + topicHistoryGeneration: 1, + updatedAt: 1 + }, + 'comparison_lookup' + ) + ] +) +stateDefinition( + MongoCollectionNames.gaspCursors, + { + remoteHost: string(2048), + topic: string(256), + state: string(64), + score: uint64, + cursorId: string(4096) + }, + ['remoteHost', 'topic', 'state'], + true, + [ + index({ network: 1, genesisHash: 1, nodeId: 1, remoteHost: 1, topic: 1 }, 'cursor_unique', { + unique: true + }) + ] +) +stateDefinition( + MongoCollectionNames.gaspGraphs, + { + graphId: string(4096), + state: { bsonType: 'string', enum: ['receiving', 'validated', 'finalized', 'discarded'] }, + reason: string(1024) + }, + ['graphId', 'state'], + true, + [ + index({ network: 1, genesisHash: 1, nodeId: 1, graphId: 1 }, 'graph_unique', { unique: true }), + index({ state: 1, updatedAt: 1 }, 'graph_state') + ] +) +stateDefinition( + MongoCollectionNames.gaspNodes, + { + graphId: string(4096), + txid: hash, + outputIndex: uint32, + payloadId: string(4096), + state: string(64) + }, + ['graphId', 'txid', 'outputIndex', 'state'], + true, + [ + index( + { network: 1, genesisHash: 1, nodeId: 1, graphId: 1, txid: 1, outputIndex: 1 }, + 'graph_node_unique', + { unique: true } + ), + index({ graphId: 1 }, 'graph_lookup') + ], + { $expr: { $lte: ['$outputIndex', '4294967295'] } } +) +stateDefinition( + MongoCollectionNames.manifestComponents, + { manifestId: string(4096), ordinal: uint64, payloadId: string(4096), kind: string(64) }, + ['manifestId', 'ordinal', 'payloadId', 'kind'], + true, + [ + index( + { network: 1, genesisHash: 1, nodeId: 1, manifestId: 1, ordinal: 1 }, + 'manifest_component_unique', + { unique: true } + ) + ] +) +for (const name of [ + MongoCollectionNames.lookupOutbox, + MongoCollectionNames.propagationOutbox +] as const) { + stateDefinition( + name, + { + eventId: string(1024), + target: string(2048), + state: string(64), + nextAttemptAt: date, + leaseUntil: date + }, + ['eventId', 'target', 'state'], + true, + [ + index({ network: 1, genesisHash: 1, nodeId: 1, eventId: 1 }, 'event_unique', { + unique: true + }), + index({ state: 1, nextAttemptAt: 1 }, 'outbox_ready') + ] + ) +} +for (const name of [MongoCollectionNames.shipRecords, MongoCollectionNames.slapRecords] as const) { + stateDefinition( + name, + { + txid: hash, + outputIndex: uint32, + domain: string(255), + topic: string(256), + service: string(256), + state: string(64) + }, + ['txid', 'outputIndex', 'domain', 'state'], + true, + [ + index( + { network: 1, genesisHash: 1, nodeId: 1, txid: 1, outputIndex: 1 }, + 'discovery_unique', + { unique: true } + ), + index({ domain: 1, topic: 1, createdAt: 1 }, 'discovery_lookup') + ], + { $expr: { $lte: ['$outputIndex', '4294967295'] } } + ) +} +stateDefinition( + MongoCollectionNames.bannedRecords, + { type: string(64), value: string(2048), bannedAt: date, state: string(64) }, + ['type', 'value', 'bannedAt', 'state'], + true, + [ + index({ network: 1, genesisHash: 1, nodeId: 1, type: 1, value: 1 }, 'ban_unique', { + unique: true + }), + index({ bannedAt: 1 }, 'ban_time') + ] +) + +export const MongoCollectionDefinitions: readonly MongoCollectionDefinition[] = definitions + +function stable(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(stable).join(',')}]` + if (value !== null && typeof value === 'object') { + const record = value as Record + return `{${Object.keys(record) + .sort() + .map(key => `${JSON.stringify(key)}:${stable(record[key])}`) + .join(',')}}` + } + return JSON.stringify(value) +} + +const schemaFingerprint = (): string => { + const source = stable( + MongoCollectionDefinitions.map(({ name, validator, indexes }) => ({ name, validator, indexes })) + ) + return createHash('sha256').update(source, 'utf8').digest('hex') +} + +export interface MongoOverlayCapability { + readonly topology: 'replica-set' + readonly schemaVersion: 1 + readonly scope: Readonly + readonly collections: readonly string[] +} + +/** + * Creates only empty Overlay-owned collections and refuses all incompatible + * pre-existing schema. It never drops, migrates, or relaxes validation. + */ +export async function bootstrapMongoOverlay( + db: Db, + scope: StorageScope +): Promise { + validChainScope(scope) + validPart(scope.nodeId) + const hello = await db.command({ hello: 1 }) + if (hello.msg === 'isdbgrid' || typeof hello.setName !== 'string' || hello.setName.length === 0) + throw new Error('Mongo Overlay storage requires an unsharded replica set') + if (hello.isWritablePrimary !== true) + throw new Error('Mongo Overlay storage requires the writable primary') + + const fingerprint = schemaFingerprint() + for (const definition of MongoCollectionDefinitions) await ensureCollection(db, definition) + await ensureGridFs(db) + await ensureLedger(db, scope, fingerprint) + await transactionalProbe(db, scope) + return { + topology: 'replica-set', + schemaVersion, + scope: { ...scope }, + collections: Object.values(MongoCollectionNames) + } +} + +async function ensureCollection(db: Db, definition: MongoCollectionDefinition): Promise { + const existing = await db.listCollections({ name: definition.name }).next() + if (existing === null) { + try { + await db.createCollection(definition.name, { + validator: definition.validator, + validationLevel: 'strict', + validationAction: 'error', + collation: { locale: 'simple' } + }) + } catch (error) { + if ((error as { code?: number }).code !== 48) throw error + } + } + const actual = await db.listCollections({ name: definition.name }).next() + if ( + actual === null || + stable(actual.options?.validator) !== stable(definition.validator) || + (actual.options?.collation !== undefined && + stable(actual.options.collation) !== stable({ locale: 'simple' })) + ) + throw new Error(`Incompatible Mongo Overlay validator for ${definition.name}`) + const collection = db.collection(definition.name) + for (const expected of definition.indexes) { + const { key, ...options } = expected + try { + await collection.createIndex(key, options) + } catch (error) { + if (![68, 85, 86].includes((error as { code?: number }).code ?? -1)) throw error + } + } + const actualIndexes = await collection.listIndexes().toArray() + for (const expected of definition.indexes) { + const actualIndex = actualIndexes.find(candidate => candidate.name === expected.name) + if ( + actualIndex === undefined || + stable(actualIndex.key) !== stable(expected.key) || + Boolean(actualIndex.unique) !== Boolean(expected.unique) || + stable(actualIndex.partialFilterExpression) !== stable(expected.partialFilterExpression) || + Boolean(actualIndex.sparse) !== Boolean(expected.sparse) || + actualIndex.expireAfterSeconds !== expected.expireAfterSeconds + ) + throw new Error(`Incompatible Mongo Overlay index for ${definition.name}:${expected.name}`) + } +} + +async function ensureGridFs(db: Db): Promise { + const filesName = `${MongoGridFsBucketName}.files` + const chunksName = `${MongoGridFsBucketName}.chunks` + for (const name of [filesName, chunksName]) { + const existing = await db.listCollections({ name }).next() + if (existing === null) { + try { + await db.createCollection(name) + } catch (error) { + if ((error as { code?: number }).code !== 48) throw error + } + } + } + await db + .collection(filesName) + .createIndex({ filename: 1, uploadDate: 1 }, { name: 'filename_1_uploadDate_1' }) + await db + .collection(chunksName) + .createIndex({ files_id: 1, n: 1 }, { unique: true, name: 'files_id_1_n_1' }) +} + +async function ensureLedger(db: Db, scope: StorageScope, fingerprint: string): Promise { + const collection = db.collection(MongoCollectionNames.schema) + const id = mongoNodeKey(scope) + const now = new Date() + try { + await collection.insertOne({ + _id: id, + schemaVersion, + network: scope.network, + genesisHash: scope.genesisHash, + nodeId: scope.nodeId, + schemaFingerprint: fingerprint, + createdAt: now, + updatedAt: now + }) + } catch (error) { + if ((error as { code?: number }).code !== 11000) throw error + } + const ledger = await collection.findOne({ _id: id }) + if ( + ledger === null || + ledger.schemaVersion !== schemaVersion || + ledger.network !== scope.network || + ledger.genesisHash !== scope.genesisHash || + ledger.nodeId !== scope.nodeId || + ledger.schemaFingerprint !== fingerprint + ) + throw new Error('Incompatible Mongo Overlay schema ledger') +} + +async function transactionalProbe(db: Db, scope: StorageScope): Promise { + const session = db.client.startSession() + try { + await session.withTransaction( + async () => { + const result = await db + .collection(MongoCollectionNames.schema) + .updateOne( + { _id: mongoNodeKey(scope), schemaVersion, schemaFingerprint: schemaFingerprint() }, + { $set: { lastTransactionProbeAt: new Date(), updatedAt: new Date() } }, + { session } + ) + if (result.matchedCount !== 1) + throw new Error('Mongo Overlay schema ledger changed during bootstrap') + }, + { readConcern: { level: 'majority' }, writeConcern: { w: 'majority', j: true } } + ) + } finally { + await session.endSession() + } +} diff --git a/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts b/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts new file mode 100644 index 000000000..8788aac15 --- /dev/null +++ b/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts @@ -0,0 +1,376 @@ +import { randomUUID } from 'node:crypto' +import { Binary, type ClientSession, type Db, type Document } from 'mongodb' +import { + admissionSemanticDigest, + type AdmissionCommitResult, + type AdmissionIdentity, + type AdmissionOperationKey, + type AdmissionReceipt, + type AdmissionReconcileResult, + type StorageScope +} from '../AdmissionStorage.js' +import { + decodeMongoUint64, + encodeMongoUint64, + MongoCollectionNames, + mongoNodeKey, + mongoRecordKey +} from './MongoSchema.js' + +export interface MongoTransactionRequest { + key: AdmissionOperationKey + identity: AdmissionIdentity + receipt: AdmissionReceipt +} + +export interface MongoTransactionOptions { + /** Total call budget, including claim, body and commit. Range 1..50000 ms. */ + timeoutMS?: number + signal?: AbortSignal +} + +export interface MongoTransactionRunnerOptions { + maxBodyAttempts?: number + maxCommitAttempts?: number + /** Persisted ownership lease; expiry permits a fenced abort, never implies one. */ + leaseMS?: number + /** Unresolved sessions are retained for same-session commit reconciliation. */ + maxRetainedSessions?: number +} + +export interface MongoTransactionContext { + readonly session: ClientSession + /** Obtain immediately before EACH database operation; do not cache or omit it. */ + options: () => { session: ClientSession; timeoutMS: number } + /** Cooperative body gate. Mongo write deadlines use timeoutMS, not signal. */ + readonly signal: AbortSignal +} + +interface Operation extends Document { + _id: string + semanticDigest: string + txid: string + state: 'pending' | 'committed' | 'aborted' + attemptId: string + leaseOwner: string + leaseToken: string + receipt?: Binary +} + +interface Attempt { + operation: Operation + session: ClientSession + phase: 'body' | 'commit' | 'unknown' + receipt: AdmissionReceipt + busy: boolean +} + +const majority = { w: 'majority' as const, j: true } + +function boundedInteger(value: number, minimum: number, maximum: number): number { + if (!Number.isSafeInteger(value) || value < minimum || value > maximum) + throw new Error('Invalid Mongo transaction bound') + return value +} + +function hasLabel(error: unknown, label: string): boolean { + return ( + typeof error === 'object' && + error !== null && + 'hasErrorLabel' in error && + typeof error.hasErrorLabel === 'function' && + error.hasErrorLabel(label) === true + ) +} + +function duplicateKey(error: unknown): boolean { + return typeof error === 'object' && error !== null && 'code' in error && error.code === 11000 +} + +class Budget { + private readonly controller = new AbortController() + private readonly timer: ReturnType + private readonly deadline: number + readonly signal: AbortSignal + + constructor(options: MongoTransactionOptions) { + const timeoutMS = boundedInteger(options.timeoutMS ?? 5000, 1, 50000) + this.deadline = performance.now() + timeoutMS + this.signal = + options.signal === undefined + ? this.controller.signal + : AbortSignal.any([this.controller.signal, options.signal]) + this.timer = setTimeout(() => this.controller.abort(new Error('Mongo transaction deadline exceeded')), timeoutMS) + this.timer.unref() + } + + remaining(): number { + this.signal.throwIfAborted() + const remaining = Math.ceil(this.deadline - performance.now()) + if (remaining <= 0) throw new Error('Mongo transaction deadline exceeded') + return remaining + } + + options(): { timeoutMS: number } { + return { timeoutMS: this.remaining() } + } + + close(): void { + clearTimeout(this.timer) + // Cached operation options must become unusable after the trusted body exits. + this.controller.abort(new Error('Mongo transaction call ended')) + } +} + +function copyReceipt(receipt: AdmissionReceipt): AdmissionReceipt { + if ( + receipt.durability !== 'atomic-local' || + typeof receipt.steak !== 'string' || + !receipt.steak.isWellFormed() || + !Array.isArray(receipt.indexes) || + receipt.indexes.some(index => typeof index.target !== 'string' || index.target.length === 0 || !index.target.isWellFormed() || (index.state !== 'visible' && index.state !== 'pending')) || + (receipt.propagation !== 'not-requested' && receipt.propagation !== 'pending') + ) throw new Error('Invalid Mongo transaction receipt') + JSON.parse(receipt.steak) + const copy: AdmissionReceipt = { + operationId: receipt.operationId, + semanticDigest: receipt.semanticDigest, + durability: 'atomic-local', + steak: receipt.steak, + indexes: receipt.indexes.map(index => ({ target: index.target, state: index.state })), + propagation: receipt.propagation + } + if (Buffer.byteLength(JSON.stringify(copy), 'utf8') > 1048576) + throw new Error('Mongo transaction receipt is too large') + return copy +} + +/** + * Database-only transaction foundation, NOT an AdmissionStorage adapter. The + * trusted body must enlist all its reads/writes using context.options(), finish + * all its work before returning, and perform no uploads, plug-ins or network I/O. + * The caller remains responsible for admission, read guards, spends and payloads. + */ +export class MongoTransactionRunner { + private readonly scope: StorageScope + private readonly owner = randomUUID() + private readonly attempts = new Map() + private readonly maxBodyAttempts: number + private readonly maxCommitAttempts: number + private readonly leaseMS: number + private readonly maxRetainedSessions: number + private closed = false + private reservations = 0 + private calls = 0 + + constructor(private readonly db: Db, scope: StorageScope, options: MongoTransactionRunnerOptions = {}) { + mongoNodeKey(scope) + this.scope = { ...scope } + this.maxBodyAttempts = boundedInteger(options.maxBodyAttempts ?? 3, 1, 10) + this.maxCommitAttempts = boundedInteger(options.maxCommitAttempts ?? 3, 1, 10) + this.leaseMS = boundedInteger(options.leaseMS ?? 30000, 1, 60000) + this.maxRetainedSessions = boundedInteger(options.maxRetainedSessions ?? 64, 1, 1024) + } + + private collection() { + return this.db.collection(MongoCollectionNames.submissionOperations) + } + + private id(key: AdmissionOperationKey): string { + if (this.closed) throw new Error('Mongo transaction runner is closed') + if (mongoNodeKey(key.scope) !== mongoNodeKey(this.scope) || !/^[0-9a-f]{64}$/.test(key.semanticDigest)) + throw new Error('Invalid Mongo transaction operation scope or digest') + return mongoRecordKey('operation', key.scope.network, key.scope.genesisHash, key.scope.nodeId, key.operationId) + } + + private async read(id: string, budget: Budget): Promise { + return await this.collection().findOne({ _id: id }, { ...budget.options(), readConcern: { level: 'majority' }, readPreference: 'primary' }) + } + + private result(operation: Operation, key: AdmissionOperationKey): AdmissionReconcileResult { + if (operation.semanticDigest !== key.semanticDigest) return { state: 'rejected', code: 'digest-mismatch' } + if (operation.state === 'committed') { + if (!(operation.receipt instanceof Binary)) throw new Error('Committed Mongo operation has no receipt') + const receipt = copyReceipt(JSON.parse(Buffer.from(operation.receipt.value()).toString('utf8')) as AdmissionReceipt) + if (receipt.operationId !== key.operationId || receipt.semanticDigest !== key.semanticDigest) + throw new Error('Corrupt Mongo operation receipt identity') + return { state: 'committed', receipt } + } + return operation.state === 'aborted' ? { state: 'aborted' } : { state: 'pending', attemptId: operation.attemptId } + } + + private fence(operation: Operation) { + return { _id: operation._id, state: 'pending' as const, semanticDigest: operation.semanticDigest, attemptId: operation.attemptId, leaseOwner: operation.leaseOwner, leaseToken: operation.leaseToken } + } + + private async release(attempt: Attempt): Promise { + this.attempts.delete(attempt.operation.attemptId) + await attempt.session.endSession() + } + + private async abort(attempt: Attempt): Promise { + try { + if (attempt.session.inTransaction()) await attempt.session.abortTransaction({ timeoutMS: 500 }) + } finally { + // This is an actual majority write to the row the transaction also writes. + // If it succeeds, a competing late commit cannot subsequently win. + try { + await this.collection().updateOne(this.fence(attempt.operation), [ + { $set: { state: 'aborted', guard: randomUUID(), updatedAt: '$$NOW' } } + ], { writeConcern: majority, timeoutMS: 1000 }) + } finally { + await this.release(attempt) + } + } + } + + private async commit(attempt: Attempt, budget: Budget): Promise { + attempt.phase = 'commit' + attempt.busy = true + try { + for (let index = 0; index < this.maxCommitAttempts; index += 1) { + try { + await attempt.session.commitTransaction({ timeoutMS: budget.remaining() }) + await this.release(attempt) + return { state: 'committed', receipt: copyReceipt(attempt.receipt) } + } catch (error) { + // Unknown wins over any accompanying transient label. Never rerun its body. + // Even an unexpected unlabeled failure after invocation stays pending. + // Reconciliation, not error wording, determines its final outcome. + void error + attempt.phase = 'unknown' + if (budget.signal.aborted) break + } + } + return { state: 'pending', attemptId: attempt.operation.attemptId } + } finally { + attempt.busy = false + } + } + + private async claim(previous: Operation, request: MongoTransactionRequest, receipt: AdmissionReceipt, budget: Budget): Promise { + if (this.attempts.size + this.reservations >= this.maxRetainedSessions) + throw new Error('Mongo unresolved transaction capacity reached') + this.reservations += 1 + try { + const attemptId = randomUUID() + const operation = await this.collection().findOneAndUpdate({ _id: previous._id, semanticDigest: request.key.semanticDigest, state: 'aborted', leaseToken: previous.leaseToken }, [{ $set: { state: 'pending', attemptId, leaseOwner: this.owner, leaseToken: encodeMongoUint64((BigInt(decodeMongoUint64(previous.leaseToken)) + BigInt(1)).toString()), guard: randomUUID(), leaseUntil: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.leaseMS } }, updatedAt: '$$NOW' } }], { ...budget.options(), writeConcern: majority, returnDocument: 'after' }) + if (operation === null) return null + const session = this.db.client.startSession() + const attempt: Attempt = { operation, session, phase: 'body', receipt, busy: true } + this.attempts.set(attemptId, attempt) + return attempt + } finally { + this.reservations -= 1 + } + } + + async run(request: MongoTransactionRequest, body: (context: MongoTransactionContext) => Promise, options: MongoTransactionOptions = {}): Promise { + request = { + key: { ...request.key, scope: { ...request.key.scope } }, + identity: { ...request.identity, scope: { ...request.identity.scope }, topics: request.identity.topics.map(topic => ({ ...topic })) }, + receipt: copyReceipt(request.receipt) + } + const id = this.id(request.key) + if (admissionSemanticDigest(request.identity) !== request.key.semanticDigest || mongoNodeKey(request.identity.scope) !== mongoNodeKey(this.scope)) + return { state: 'rejected', code: 'digest-mismatch' } + const receipt = copyReceipt(request.receipt) + if (receipt.operationId !== request.key.operationId || receipt.semanticDigest !== request.key.semanticDigest) + return { state: 'rejected', code: 'invalid-plan' } + const budget = new Budget(options) + this.calls += 1 + try { + for (let bodyIndex = 0; bodyIndex < this.maxBodyAttempts; bodyIndex += 1) { + let previous = await this.read(id, budget) + if (previous === null) { + try { + await this.collection().insertOne({ _id: id, schemaVersion: 1, ...this.scope, operationId: request.key.operationId, semanticDigest: request.key.semanticDigest, txid: request.identity.txid, state: 'aborted', attemptId: randomUUID(), leaseOwner: this.owner, leaseToken: encodeMongoUint64('0'), leaseUntil: new Date(0), guard: randomUUID(), createdAt: new Date(), updatedAt: new Date() }, { ...budget.options(), writeConcern: majority }) + } catch (error) { + if (!duplicateKey(error)) throw error + } + previous = await this.read(id, budget) + if (previous === null) throw new Error('Mongo operation claim was not visible') + } + const existing = this.result(previous, request.key) + if (existing.state !== 'aborted') return existing + const attempt = await this.claim(previous, request, receipt, budget) + if (attempt === null) { + const winner = await this.read(id, budget) + if (winner === null) throw new Error('Mongo operation claim disappeared') + const result = this.result(winner, request.key) + if (result.state !== 'aborted') return result + continue + } + const { session, operation } = attempt + let bodyActive = true + const context: MongoTransactionContext = { session, signal: budget.signal, options: () => { + if (!bodyActive || !session.inTransaction()) throw new Error('Mongo transaction body is no longer active') + return { session, ...budget.options() } + } } + try { + session.startTransaction({ readConcern: { level: 'snapshot' }, writeConcern: majority, readPreference: 'primary', maxCommitTimeMS: budget.remaining() }) + const guarded = await this.collection().updateOne({ ...this.fence(operation), $expr: { $gt: ['$leaseUntil', '$$NOW'] } }, { $set: { guard: randomUUID() } }, context.options()) + if (guarded.modifiedCount !== 1) throw new Error('Mongo transaction ownership lost') + // Never end a session while its body is still running. Trusted bodies + // await every bounded database operation and observe the context gate. + await body(context) + const saved = await this.collection().updateOne(this.fence(operation), { $set: { state: 'committed', receipt: new Binary(Buffer.from(JSON.stringify(receipt), 'utf8')), guard: randomUUID() }, $currentDate: { updatedAt: true } }, context.options()) + if (saved.modifiedCount !== 1) throw new Error('Mongo transaction ownership lost') + bodyActive = false + return await this.commit(attempt, budget) + } catch (error) { + bodyActive = false + await this.abort(attempt) + if (!hasLabel(error, 'TransientTransactionError') || bodyIndex + 1 >= this.maxBodyAttempts) throw error + } finally { + bodyActive = false + attempt.busy = false + } + } + throw new Error('Mongo transaction attempt limit reached') + } finally { + this.calls -= 1 + budget.close() + } + } + + async reconcile(key: AdmissionOperationKey, attemptId?: string, options: MongoTransactionOptions = {}): Promise { + key = { ...key, scope: { ...key.scope } } + const id = this.id(key) + const budget = new Budget(options) + this.calls += 1 + try { + let operation = await this.read(id, budget) + // Absence does not prove abort, including after process restart. + if (operation === null) return { state: 'pending', attemptId: attemptId ?? 'unlocated' } + const result = this.result(operation, key) + if (result.state !== 'pending') { + const retained = this.attempts.get(operation.attemptId) + if (retained !== undefined && !retained.busy) await this.release(retained) + return result + } + if (attemptId !== undefined && attemptId !== operation.attemptId) return { state: 'pending', attemptId } + const retained = this.attempts.get(operation.attemptId) + if (retained !== undefined) { + if (retained.busy || retained.phase === 'body') return result + return await this.commit(retained, budget) + } + // A successful majority CAS, not elapsed time or a missing record, proves + // an orphan cannot commit: its transaction must write this same row first. + const aborted = await this.collection().findOneAndUpdate({ ...this.fence(operation), $expr: { $lte: ['$leaseUntil', '$$NOW'] } }, [{ $set: { state: 'aborted', guard: randomUUID(), updatedAt: '$$NOW' } }], { ...budget.options(), writeConcern: majority, returnDocument: 'after' }) + if (aborted !== null) return { state: 'aborted' } + operation = await this.read(id, budget) + return operation === null ? result : this.result(operation, key) + } finally { + this.calls -= 1 + budget.close() + } + } + + /** Shutdown only. Persisted receipt/fenced orphan reconciliation survives it. */ + async close(): Promise { + if (this.calls > 0 || this.reservations > 0 || [...this.attempts.values()].some(attempt => attempt.busy)) throw new Error('Cannot close Mongo transaction runner during a call') + this.closed = true + await Promise.all([...this.attempts.values()].map(async attempt => await this.release(attempt))) + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32cc1896c..caba6275a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1220,6 +1220,12 @@ importers: jest: specifier: ^30.4.2 version: 30.4.2(@types/node@26.1.2)(ts-node@10.9.2(@types/node@26.1.2)(@typescript/typescript6@6.0.2)) + mongodb: + specifier: 7.5.0 + version: 7.5.0 + mongodb-memory-server: + specifier: ^11.2.0 + version: 11.2.0 oxlint: specifier: ^1.76.0 version: 1.76.0 diff --git a/specs/overlay/mongo-v1.md b/specs/overlay/mongo-v1.md new file mode 100644 index 000000000..5bfb02a43 --- /dev/null +++ b/specs/overlay/mongo-v1.md @@ -0,0 +1,123 @@ +# Overlay Mongo foundation v1 + +## Status and boundary + +This is an additive, opt-in MongoDB storage foundation for `@bsv/overlay`. +It supplies schema bootstrap, scoped keys, content-addressed payload primitives, +and durable read-guard primitives. It does **not** select MongoDB by default, +connect MongoDB to `Engine`, implement `AdmissionStorage`, or make an admission +receipt claim. The existing Engine, Knex, and injected-storage paths retain +their current behavior. + +MongoDB is an optional peer dependency. An application that imports a Mongo +entry point must install a compatible driver explicitly: + +```sh +npm install @bsv/overlay mongodb@^7.5.0 +``` + +The root package entry deliberately does not load the driver. Mongo consumers +use the exported deep modules, for example +`@bsv/overlay/storage/mongo/MongoSchema` and +`@bsv/overlay/storage/mongo/MongoPayloadStore`. A packed-consumer check must +cover those exact ESM and CommonJS imports as well as a root import without the +optional peer installed. + +## Deployment and schema + +`bootstrapMongoOverlay` creates or verifies only Overlay-owned collections, +including the `overlayPayloads` GridFS files and chunks collections. It +refuses incompatible validators, indexes, schema fingerprints, a sharded +deployment, and a non-primary bootstrap connection; it never drops, migrates, +or relaxes existing records. Bootstrap also makes a majority+journaled +transaction probe. + +The supported operating profile is a dedicated, unsharded three-member replica +set for an Overlay deployment. The bootstrap check establishes that the server +is a writable replica-set primary; operators remain responsible for ensuring +the three-member topology, TLS, authentication, backups, and capacity. MongoDB +transactions require a replica set or sharded cluster, and transaction runtime +and resource limits still apply. See MongoDB's [transaction production +considerations](https://www.mongodb.com/docs/manual/core/transactions-production-consideration/). + +Every Overlay-owned row has a schema version and bounded identity parts. +Chain-wide records include `network` and `genesisHash`; node-owned records add +`nodeId`. Record identifiers use an unambiguous framed encoding, rather than a +delimiter-joined string. Exact wire integers are canonical decimal strings: +the schema helpers reject non-canonical values and preserve uint64 values +without passing them through an unsafe JavaScript number. Output indices have +the stricter uint32 range. Collection validators and indexes bound row shape, +field length, array size, receipt size, and indexed identity fields so an +unbounded owner record is not silently created. MongoDB's BSON document ceiling +is 16 MiB; schema bounds remain necessary even below that ceiling. See +[MongoDB limits](https://www.mongodb.com/docs/manual/reference/limits/#bson-documents). + +## Payload publication, references, and collection + +`MongoPayloadStore` stores content by chain scope, kind, and SHA-256 digest. +It validates canonical declared length, digest, raw-transaction identity where +provided, and configured application bounds before publishing. Small content is +kept inline under a conservative ceiling. Larger content follows this sequence: + +1. Reserve a fenced `uploading` payload row. Reservation lease comparisons use + Mongo server time. +2. Stream bytes to a GridFS file marked `staged`, then verify the observed + length and digest. +3. Mark that physical GridFS file `published` with its owner and fencing-token + metadata. +4. Compare-and-set the small payload row to `ready`. Only a ready row can be + referenced. + +This order is intentional: GridFS cannot participate in multi-document +transactions, so a transaction must only pin already-published content. See +the MongoDB [GridFS documentation](https://www.mongodb.com/docs/manual/core/gridfs/). +The store's recovery operation can either finish a verifiable expired upload or +retire only the fenced owner's incomplete file; it does not assume that an +unexplained owner record is disposable. + +Reference creation is a caller-session operation. It conditionally updates the +same `ready` payload row before inserting the reference; a prior snapshot read +is not accepted as the guard. Garbage collection counts live references and +claims `ready` to `deleting` in the caller transaction. Once the claim is +visible, a new reference cannot pass the same-row `ready` guard. Physical +GridFS deletion and `deleted` finalization resume separately and are safe to +retry after a process failure. Pins are references and are the only reference +kind allowed to expire. + +Payload and reference operations accept bounded timeout and cancellation +controls where they operate in a caller's admission body. Neither payload +publication nor collection runs verifier logic, network activity, uploads, or +plugin callbacks inside an admission transaction. + +## Admission transaction integration is pending + +The foundation is deliberately smaller than the v1 admission contract. A later +Engine/adapter integration must compose the guarded payload primitives with the +whole admission operation in one short transaction. Until that integration and +its conformance evidence land, no Mongo method is an `AdmissionStorage` +implementation and no saved STEAK/receipt is exposed as a durable admission +acknowledgment. + +The pending transaction core must use a finite retry budget and distinguish +MongoDB's labels on the **same session and operation identity**: rerun a +`TransientTransactionError` body only under that identity, and reconcile an +`UnknownTransactionCommitResult` before a new body may run. Its completion +record must be majority+journal persisted with the exact receipt. Expired +orphan cleanup must use a compare-and-set proof rather than treating absence or +an ambiguous commit as an abort. MongoDB documents the retry-label behavior in +its [transaction API guidance](https://www.mongodb.com/docs/manual/core/transactions-in-applications/). + +That integration must keep external indexes and propagation asynchronous: +persist an idempotent outbox intent in the admission transaction, then let a +leased worker deliver it. A callback may participate only after it declares +replay safety, reconciliation, cancellation, and bounded work. It must never +be invoked in the Mongo admission transaction. + +## Compatibility and operations + +There is no SQL-to-Mongo migration, dual-write mode, shared concurrent-writer +topology, or default-adapter switch in this foundation. Do not point existing +SQL and Mongo writers at the same logical Overlay authority. Operators should +run Mongo bootstrap and packed-consumer verification before an explicit future +adapter activation; that activation will need its own versioned migration and +recovery plan. From b25a0192119614f5e4b704c6f9840d3dfd5a7e48 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 14:57:38 -0500 Subject: [PATCH 007/168] fix(sdk): close C02/C03 evidence coordinator review findings Preserve per-entry script results so a rejected sibling cannot poison a shared ancestor. LocalChainTracker tokens use participating sources only and fail closed on missing identity. Reset ownership is rechecked after dispose before destructive hooks. Built-in remote ChainTracks clients advertise reorg-event capability explicitly. Services.getChainTracker publishes one coalesced wrapper before yielding. --- docs/packages/sdk/bsv-sdk.md | 2 +- docs/reference/package-api-migrations.md | 88 +- docs/reference/stack-facts.md | 2 +- governance/package-release-notes.json | 18 +- governance/repository-health/baselines.json | 2 +- packages/sdk/README.md | 6 + packages/sdk/docs/reference/overlay-tools.md | 819 +++++-- packages/sdk/docs/reference/transaction.md | 1963 ++++++++++++----- packages/sdk/docs/transaction-evidence.md | 160 ++ packages/sdk/package.json | 2 +- .../sdk/src/overlay-tools/LookupResolver.ts | 198 +- .../__tests/LookupResolver.evidence.test.ts | 232 ++ packages/sdk/src/transaction/ChainTracker.ts | 19 +- .../sdk/src/transaction/EvidenceScriptWork.ts | 291 +++ packages/sdk/src/transaction/Transaction.ts | 76 +- .../src/transaction/TransactionEvidence.ts | 163 ++ .../TransactionEvidenceCoordinator.ts | 583 +++++ .../TransactionEvidenceCoordinator.test.ts | 1085 +++++++++ packages/sdk/src/transaction/index.ts | 26 +- packages/wallet/wallet-toolbox/CHANGELOG.md | 21 +- packages/wallet/wallet-toolbox/README.md | 21 +- .../wallet/wallet-toolbox/client/README.md | 14 +- .../docs/identity-verification.md | 252 ++- .../wallet/wallet-toolbox/mobile/README.md | 14 +- packages/wallet/wallet-toolbox/src/Wallet.ts | 80 +- .../Wallet.identityVerification.test.ts | 72 +- .../wallet-toolbox/src/services/Services.ts | 40 +- .../src/services/__tests/verifyBeef.test.ts | 29 + .../services/chaintracker/BHServiceClient.ts | 2 + .../chaintracker/ChaintracksChainTracker.ts | 189 +- .../chaintracker/LocalChainTracker.ts | 210 +- .../__tests/ChaintracksChainTracker.test.ts | 130 +- .../__tests/LocalChainTracker.test.ts | 267 +++ .../chaintracks/Api/ChaintracksClientApi.ts | 7 + .../chaintracks/ChaintracksServiceClient.ts | 2 + .../chaintracks/GoChaintracksServiceClient.ts | 2 + .../__tests__/identityVerification.test.ts | 32 +- .../src/utility/identityUtils.ts | 233 +- .../src/utility/verifyOverlayOutput.ts | 79 +- 39 files changed, 6252 insertions(+), 1179 deletions(-) create mode 100644 packages/sdk/docs/transaction-evidence.md create mode 100644 packages/sdk/src/overlay-tools/__tests/LookupResolver.evidence.test.ts create mode 100644 packages/sdk/src/transaction/EvidenceScriptWork.ts create mode 100644 packages/sdk/src/transaction/TransactionEvidence.ts create mode 100644 packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts create mode 100644 packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts diff --git a/docs/packages/sdk/bsv-sdk.md b/docs/packages/sdk/bsv-sdk.md index 14ef20ca4..88203bf6c 100644 --- a/docs/packages/sdk/bsv-sdk.md +++ b/docs/packages/sdk/bsv-sdk.md @@ -3,7 +3,7 @@ id: bsv-sdk title: '@bsv/sdk' kind: package domain: sdk -version: '2.4.2' +version: '2.5.0' npm: '@bsv/sdk' last_updated: '2026-08-26' last_verified: '2026-08-26' diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 7c20e6db3..09730c768 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -23,42 +23,42 @@ and clean-consumer tests remain the executable type authority. ## Current release boundary -| Package | npm baseline | Source | Candidate | API | Migration | -| --------------------------------- | ------------ | -------- | --------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `@bsv/402-pay` | `0.2.1` | `0.2.5` | patch | [API and usage](../packages/middleware/402-pay.md) | No consumer migration is required; client and server exports, payment protocol behavior, and runtime defaults are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/air-gap` | `0.0.0` | `0.1.2` | minor | [API and usage](../packages/helpers/air-gap.md) | No consumer migration is required; this is the first published release of a new package with no prior public API. The experimental pre-release framing that circulated on the unmerged feature branch is not accepted by the v1 decoder. | -| `@bsv/amountinator` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/helpers/amountinator.md) | No consumer migration is required; this is a backward-compatible patch candidate. | -| `@bsv/auth` | `0.1.1` | `0.1.4` | patch | [API and usage](../packages/middleware/auth.md) | No consumer migration is required; authentication APIs, wire behavior, and runtime defaults are unchanged. | -| `@bsv/auth-express-middleware` | `2.2.0` | `2.2.3` | patch | [API and usage](../packages/middleware/auth-express-middleware.md) | No API migration is required. Upgrade to @bsv/sdk 2.4.1 or later for the shared byte-boundary contract. Generic signed application-body canonicalization remains unchanged so old and new peers verify identical bytes. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/authsocket` | `2.1.1` | `2.1.7` | patch | [API and usage](../packages/messaging/authsocket.md) | No API migration is required. Existing event data, including numeric-key objects under byte-like names, is unchanged; typed payment protocols recover historical byte objects at their explicit fields. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/authsocket-client` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/messaging/authsocket-client.md) | No API migration is required. Existing event data, including numeric-key objects under byte-like names, is unchanged; typed payment protocols recover historical byte objects at their explicit fields. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | -| `@bsv/btms` | `1.1.1` | `1.2.2` | minor | [API and usage](../packages/wallet/btms.md) | Existing local, mainnet, testnet, and number-array behavior is unchanged. TTN consumers select networkPreset teratestnet; all consumers should upgrade to @bsv/sdk 2.4.1 or later for byte-boundary compatibility. | -| `@bsv/btms-permission-module` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/wallet/btms-permission-module.md) | No consumer migration is required; permission-module APIs and token semantics are unchanged. | -| `@bsv/chirp` | `0.0.0` | `0.1.1` | minor | [API and usage](../packages/network/chirp.md) | No consumer migration is required; this is the first release of a new additive package. Existing @bsv/sdk StorageUploader, StorageDownloader, StorageUtils, UHRP identifiers, overlays, and server routes remain unchanged. BRC-167 remains authoritative if the implementation and standard differ. | -| `@bsv/did` | `0.2.1` | `0.2.5` | patch | [API and usage](../packages/helpers/did.md) | No consumer migration is required; DID APIs, encodings, credential behavior, and supported import forms are unchanged. | -| `@bsv/did-client` | `1.2.1` | `1.3.1` | minor | [API and usage](../packages/helpers/did-client.md) | Existing local, mainnet, and testnet behavior is unchanged. TTN consumers select networkPreset teratestnet and use @bsv/sdk 2.4 or later. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | -| `@bsv/ecpm-permission-module` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/wallet/ecpm-permission-module.md) | No existing consumer migration is required; this is the first release. Wallet hosts register the module under the ecpm scheme and supply their ordinary key deriver, authorization handler, and optional privileged key provider. | -| `@bsv/fund-wallet` | `1.4.1` | `1.4.4` | patch | [API and usage](../packages/helpers/fund-wallet.md) | No consumer migration is required; wallet funding APIs and transaction behavior are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | -| `@bsv/message-box-client` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | -| `@bsv/overlay` | `2.2.1` | `2.3.1` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.1` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-express` | `2.5.0` | `2.6.1` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-topics` | `1.6.10` | `1.7.1` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. | -| `@bsv/paymail` | `2.4.2` | `2.4.7` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. Consumers of the former bundled Money Button or Tokenized specification documents must follow the authoritative links in docs/specs/README.md. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/payment-express-middleware` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/sdk` | `2.4.0` | `2.4.2` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. | -| `@bsv/simple` | `0.4.1` | `0.5.3` | minor | [API and usage](../packages/helpers/simple.md) | Existing overlay configurations and number-array behavior are unchanged. TTN consumers select network teratestnet; all consumers should upgrade to @bsv/sdk 2.4.2 or later. The Message Box Client dependency advance requires no helper API migration and preserves the default HTTP and live-socket behavior. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/. | -| `@bsv/templates` | `1.9.1` | `1.10.1` | minor | [API and usage](../packages/helpers/templates.md) | No existing consumer migration is required; existing template APIs and generated scripts are unchanged. New R1K1Wallet consumers await lock(), retain each private 32-byte salt, and provide a PIV signer that signs the supplied digest directly without hashing it again. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/teranode-listener` | `1.1.1` | `1.1.5` | patch | [API and usage](../packages/network/teranode-listener.md) | No consumer migration is required; listener APIs, topics, and network configuration are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/verifast` | `0.3.0` | `0.3.5` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. Keep THIRD_PARTY_NOTICES.md and LICENSES/ with every JavaScript and WebAssembly distribution. | -| `@bsv/wallet-helper` | `0.1.1` | `0.1.7` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/wallet-relay` | `0.2.2` | `0.3.6` | minor | [API and usage](../packages/wallet/wallet-relay.md) | No wallet RPC migration is required; upgrade to @bsv/sdk 2.4.1 or later. Existing relay sessions and number arrays remain valid, and host applications continue to provide their matching Express runtime and type graph. | -| `@bsv/wallet-toolbox` | `2.10.4` | `2.11.1` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. | -| `@bsv/wallet-toolbox-client` | `2.10.4` | `2.11.1` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. | -| `@bsv/wallet-toolbox-mobile` | `2.10.4` | `2.11.1` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. | -| `create-bsv-app` | `1.0.2` | `1.1.1` | minor | [API and usage](../packages/helpers/create-bsv-app.md) | Existing mainnet and testnet scaffolds are unchanged. New TTN projects pass --network ttn or select TerraTestNet in the configurator. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| Package | npm baseline | Source | Candidate | API | Migration | +| --------------------------------- | ------------ | -------- | --------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `@bsv/402-pay` | `0.2.1` | `0.2.5` | patch | [API and usage](../packages/middleware/402-pay.md) | No consumer migration is required; client and server exports, payment protocol behavior, and runtime defaults are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/air-gap` | `0.0.0` | `0.1.2` | minor | [API and usage](../packages/helpers/air-gap.md) | No consumer migration is required; this is the first published release of a new package with no prior public API. The experimental pre-release framing that circulated on the unmerged feature branch is not accepted by the v1 decoder. | +| `@bsv/amountinator` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/helpers/amountinator.md) | No consumer migration is required; this is a backward-compatible patch candidate. | +| `@bsv/auth` | `0.1.1` | `0.1.4` | patch | [API and usage](../packages/middleware/auth.md) | No consumer migration is required; authentication APIs, wire behavior, and runtime defaults are unchanged. | +| `@bsv/auth-express-middleware` | `2.2.0` | `2.2.3` | patch | [API and usage](../packages/middleware/auth-express-middleware.md) | No API migration is required. Upgrade to @bsv/sdk 2.4.1 or later for the shared byte-boundary contract. Generic signed application-body canonicalization remains unchanged so old and new peers verify identical bytes. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/authsocket` | `2.1.1` | `2.1.7` | patch | [API and usage](../packages/messaging/authsocket.md) | No API migration is required. Existing event data, including numeric-key objects under byte-like names, is unchanged; typed payment protocols recover historical byte objects at their explicit fields. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/authsocket-client` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/messaging/authsocket-client.md) | No API migration is required. Existing event data, including numeric-key objects under byte-like names, is unchanged; typed payment protocols recover historical byte objects at their explicit fields. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | +| `@bsv/btms` | `1.1.1` | `1.2.2` | minor | [API and usage](../packages/wallet/btms.md) | Existing local, mainnet, testnet, and number-array behavior is unchanged. TTN consumers select networkPreset teratestnet; all consumers should upgrade to @bsv/sdk 2.4.1 or later for byte-boundary compatibility. | +| `@bsv/btms-permission-module` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/wallet/btms-permission-module.md) | No consumer migration is required; permission-module APIs and token semantics are unchanged. | +| `@bsv/chirp` | `0.0.0` | `0.1.1` | minor | [API and usage](../packages/network/chirp.md) | No consumer migration is required; this is the first release of a new additive package. Existing @bsv/sdk StorageUploader, StorageDownloader, StorageUtils, UHRP identifiers, overlays, and server routes remain unchanged. BRC-167 remains authoritative if the implementation and standard differ. | +| `@bsv/did` | `0.2.1` | `0.2.5` | patch | [API and usage](../packages/helpers/did.md) | No consumer migration is required; DID APIs, encodings, credential behavior, and supported import forms are unchanged. | +| `@bsv/did-client` | `1.2.1` | `1.3.1` | minor | [API and usage](../packages/helpers/did-client.md) | Existing local, mainnet, and testnet behavior is unchanged. TTN consumers select networkPreset teratestnet and use @bsv/sdk 2.4 or later. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | +| `@bsv/ecpm-permission-module` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/wallet/ecpm-permission-module.md) | No existing consumer migration is required; this is the first release. Wallet hosts register the module under the ecpm scheme and supply their ordinary key deriver, authorization handler, and optional privileged key provider. | +| `@bsv/fund-wallet` | `1.4.1` | `1.4.4` | patch | [API and usage](../packages/helpers/fund-wallet.md) | No consumer migration is required; wallet funding APIs and transaction behavior are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | +| `@bsv/message-box-client` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | +| `@bsv/overlay` | `2.2.1` | `2.3.1` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.1` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/overlay-express` | `2.5.0` | `2.6.1` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/overlay-topics` | `1.6.10` | `1.7.1` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. | +| `@bsv/paymail` | `2.4.2` | `2.4.7` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. Consumers of the former bundled Money Button or Tokenized specification documents must follow the authoritative links in docs/specs/README.md. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/payment-express-middleware` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/sdk` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/sdk/bsv-sdk.md) | No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. Existing Transaction.verify and resolver call shapes remain unchanged. New coordinator callers supply an independently maintained ChainTracker, explicit chain namespace and semantic policy id, handle bounded limit/timeout outcomes, and dispose sessions when finished. Optional ChainTracker context markers and cancellation signals are additive. No wire or storage migration is required. Coordinate resolver evidenceLimits with downstream evidence admission budgets when increasing them; limit events mean partial bounded intake, not invalid transaction evidence. | +| `@bsv/simple` | `0.4.1` | `0.5.3` | minor | [API and usage](../packages/helpers/simple.md) | Existing overlay configurations and number-array behavior are unchanged. TTN consumers select network teratestnet; all consumers should upgrade to @bsv/sdk 2.4.2 or later. The Message Box Client dependency advance requires no helper API migration and preserves the default HTTP and live-socket behavior. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/. | +| `@bsv/templates` | `1.9.1` | `1.10.1` | minor | [API and usage](../packages/helpers/templates.md) | No existing consumer migration is required; existing template APIs and generated scripts are unchanged. New R1K1Wallet consumers await lock(), retain each private 32-byte salt, and provide a PIV signer that signs the supplied digest directly without hashing it again. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/teranode-listener` | `1.1.1` | `1.1.5` | patch | [API and usage](../packages/network/teranode-listener.md) | No consumer migration is required; listener APIs, topics, and network configuration are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/verifast` | `0.3.0` | `0.3.5` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. Keep THIRD_PARTY_NOTICES.md and LICENSES/ with every JavaScript and WebAssembly distribution. | +| `@bsv/wallet-helper` | `0.1.1` | `0.1.7` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/wallet-relay` | `0.2.2` | `0.3.6` | minor | [API and usage](../packages/wallet/wallet-relay.md) | No wallet RPC migration is required; upgrade to @bsv/sdk 2.4.1 or later. Existing relay sessions and number arrays remain valid, and host applications continue to provide their matching Express runtime and type graph. | +| `@bsv/wallet-toolbox` | `2.10.4` | `2.11.1` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.5.0 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. Identity RPC, trust filtering, local contacts and stored data remain unchanged; no additional migration is needed. The unpublished 2.11.1 candidate aggregates C01/C02/C03 changes against published 2.10.4. New evidence limits are local admission policy, not consensus limits; applications using the generic coordinator can configure them explicitly. Identity lookup propagates typed limit/timeout failures instead of returning a definitive empty result when bounded work cannot complete. Direct utility callers can pass matching queryOverlayEvidence and IdentityEvidenceVerifier byte limits. ChainTracks adapters bracket validation with fresh tips and local reorg/reset epochs where supported; remote observations are not atomic snapshots. | +| `@bsv/wallet-toolbox-client` | `2.10.4` | `2.11.1` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.5.0 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. Identity RPC, trust filtering, local contacts and stored data remain unchanged; no additional migration is needed. The unpublished 2.11.1 candidate aggregates C01/C02/C03 changes against published 2.10.4. New evidence limits are local admission policy, not consensus limits; applications using the generic coordinator can configure them explicitly. Identity lookup propagates typed limit/timeout failures instead of returning a definitive empty result when bounded work cannot complete. Direct utility callers can pass matching queryOverlayEvidence and IdentityEvidenceVerifier byte limits. ChainTracks adapters bracket validation with fresh tips and local reorg/reset epochs where supported; remote observations are not atomic snapshots. | +| `@bsv/wallet-toolbox-mobile` | `2.10.4` | `2.11.1` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.5.0 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. Identity RPC, trust filtering, local contacts and stored data remain unchanged; no additional migration is needed. The unpublished 2.11.1 candidate aggregates C01/C02/C03 changes against published 2.10.4. New evidence limits are local admission policy, not consensus limits; applications using the generic coordinator can configure them explicitly. Identity lookup propagates typed limit/timeout failures instead of returning a definitive empty result when bounded work cannot complete. Direct utility callers can pass matching queryOverlayEvidence and IdentityEvidenceVerifier byte limits. ChainTracks adapters bracket validation with fresh tips and local reorg/reset epochs where supported; remote observations are not atomic snapshots. | +| `create-bsv-app` | `1.0.2` | `1.1.1` | minor | [API and usage](../packages/helpers/create-bsv-app.md) | Existing mainnet and testnet scaffolds are unchanged. New TTN projects pass --network ttn or select TerraTestNet in the configurator. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | `none` means the source manifest matches the recorded npm baseline. Any other value is an unpublished candidate. Publication, tags, releases, registry @@ -359,8 +359,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/sdk/bsv-sdk.md](../packages/sdk/bsv-sdk.md) - Source: [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) -- Release note: Adds one strict BRC-100 byte-boundary contract and secure React Native bridge validation, and completes incorporated-code compliance with hash-pinned provenance, exact MIT/ISC/BSD/Apache notices, source lineage markers, package payloads, a retained UMD banner, SBOM components, and release enforcement. Removes a broken copied Swagger UI scaffold and screenshot; the first-party OpenAPI source remains available for use with any viewer. Standardizes first-party author metadata on the current BSV Association name. -- Migration: No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. +- Release note: Adds one strict BRC-100 byte-boundary contract and secure React Native bridge validation, and completes incorporated-code compliance with hash-pinned provenance, exact MIT/ISC/BSD/Apache notices, source lineage markers, package payloads, a retained UMD banner, SBOM components, and release enforcement. Removes a broken copied Swagger UI scaffold and screenshot; the first-party OpenAPI source remains available for use with any viewer. Standardizes first-party author metadata on the current BSV Association name. Adds a generic bounded transaction-evidence coordinator with derived-txid work sharing, alternate proof candidates, exact script-work reuse, canonical dependency rechecks, context fencing and reference-counted cancellation. Adds optional pre-aggregation lookup evidence receipts while preserving legacy resolver defaults. Adds optional fresh canonical-context tokens, with explicit limits for legacy tracker and remote ABA assurance. +- Migration: No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. Existing Transaction.verify and resolver call shapes remain unchanged. New coordinator callers supply an independently maintained ChainTracker, explicit chain namespace and semantic policy id, handle bounded limit/timeout outcomes, and dispose sessions when finished. Optional ChainTracker context markers and cancellation signals are additive. No wire or storage migration is required. Coordinate resolver evidenceLimits with downstream evidence admission budgets when increasing them; limit events mean partial bounded intake, not invalid transaction evidence. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | @@ -520,8 +520,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox.md](../packages/wallet/wallet-toolbox.md) - Source: [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) -- Release note: Adds the built-in BRC-177 noSend-expiry reference implementation with exact prefunding, durable pre-signed reclaim, atomic active-storage monitoring, backoff-controlled recovery, cross-device lifecycle synchronization, and proof-finalized race handling. Adds opt-in prepared BEEF storage for Knex-backed normal createAction funding: verified, checksummed proof closures are persisted after foreground completion and reused on later hits, while broad lookups, misses, and cache failures retain the canonical path. Reads, writes, bounded queueing, and gradual backfill default off; reorganizations stale derived rows and fence in-flight cross-process writes with a database proof epoch. Also adds the optional semantic handleRequest hook for BRC-98/99/111 permission modules, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes, retains BRC-95/BRC-100 compatibility and stable bounded pagination, removes the obsolete JSight application bundle, and preserves the earlier Open BSV grant. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. -- Migration: Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. +- Release note: Adds the built-in BRC-177 noSend-expiry reference implementation with exact prefunding, durable pre-signed reclaim, atomic active-storage monitoring, backoff-controlled recovery, cross-device lifecycle synchronization, and proof-finalized race handling. Adds opt-in prepared BEEF storage for Knex-backed normal createAction funding: verified, checksummed proof closures are persisted after foreground completion and reused on later hits, while broad lookups, misses, and cache failures retain the canonical path. Reads, writes, bounded queueing, and gradual backfill default off; reorganizations stale derived rows and fence in-flight cross-process writes with a database proof epoch. Also adds the optional semantic handleRequest hook for BRC-98/99/111 permission modules, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes, retains BRC-95/BRC-100 compatibility and stable bounded pagination, removes the obsolete JSight application bundle, and preserves the earlier Open BSV grant. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. Shares bounded transaction and identity cryptographic work across final discovery calls, retains alternate receipts before resolver deduplication, rechecks current canonical anchors, and fixes stale Chaintracks height-root reuse with provider/mode context fencing. +- Migration: Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.5.0 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. Identity RPC, trust filtering, local contacts and stored data remain unchanged; no additional migration is needed. The unpublished 2.11.1 candidate aggregates C01/C02/C03 changes against published 2.10.4. New evidence limits are local admission policy, not consensus limits; applications using the generic coordinator can configure them explicitly. Identity lookup propagates typed limit/timeout failures instead of returning a definitive empty result when bounded work cannot complete. Direct utility callers can pass matching queryOverlayEvidence and IdentityEvidenceVerifier byte limits. ChainTracks adapters bracket validation with fresh tips and local reorg/reset epochs where supported; remote observations are not atomic snapshots. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ---------------------------------------------------- | -------------------------- | @@ -534,8 +534,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-client.md](../packages/wallet/wallet-toolbox-client.md) - Source: [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) -- Release note: Adds the built-in browser BRC-177 noSend-expiry signer, IndexedDB schema version 5 lifecycle state, remote storage capability negotiation, default monitor coordination, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while IndexedDB remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, stable IndexedDB totals, current browser compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. -- Migration: Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. +- Release note: Adds the built-in browser BRC-177 noSend-expiry signer, IndexedDB schema version 5 lifecycle state, remote storage capability negotiation, default monitor coordination, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while IndexedDB remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, stable IndexedDB totals, current browser compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. Shares bounded transaction and identity cryptographic work across final discovery calls, retains alternate receipts before resolver deduplication, rechecks current canonical anchors, and fixes stale Chaintracks height-root reuse with provider/mode context fencing. +- Migration: Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.5.0 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. Identity RPC, trust filtering, local contacts and stored data remain unchanged; no additional migration is needed. The unpublished 2.11.1 candidate aggregates C01/C02/C03 changes against published 2.10.4. New evidence limits are local admission policy, not consensus limits; applications using the generic coordinator can configure them explicitly. Identity lookup propagates typed limit/timeout failures instead of returning a definitive empty result when bounded work cannot complete. Direct utility callers can pass matching queryOverlayEvidence and IdentityEvidenceVerifier byte limits. ChainTracks adapters bracket validation with fresh tips and local reorg/reset epochs where supported; remote observations are not atomic snapshots. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | @@ -546,8 +546,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-mobile.md](../packages/wallet/wallet-toolbox-mobile.md) - Source: [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) -- Release note: Adds the built-in mobile BRC-177 noSend-expiry signer, remote storage capability negotiation, default-monitor ownership coordination across restarts and devices, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback for React Native runtimes without WebAssembly, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while mobile storage remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, current mobile compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. -- Migration: Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. +- Release note: Adds the built-in mobile BRC-177 noSend-expiry signer, remote storage capability negotiation, default-monitor ownership coordination across restarts and devices, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback for React Native runtimes without WebAssembly, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while mobile storage remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, current mobile compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. Shares bounded transaction and identity cryptographic work across final discovery calls, retains alternate receipts before resolver deduplication, rechecks current canonical anchors, and fixes stale Chaintracks height-root reuse with provider/mode context fencing. +- Migration: Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.5.0 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. Identity RPC, trust filtering, local contacts and stored data remain unchanged; no additional migration is needed. The unpublished 2.11.1 candidate aggregates C01/C02/C03 changes against published 2.10.4. New evidence limits are local admission policy, not consensus limits; applications using the generic coordinator can configure them explicitly. Identity lookup propagates typed limit/timeout failures instead of returning a definitive empty result when bounded work cannot complete. Direct utility callers can pass matching queryOverlayEvidence and IdentityEvidenceVerifier byte limits. ChainTracks adapters bracket validation with fresh tips and local reorg/reset epochs where supported; remote observations are not atomic snapshots. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 45b71107d..46d312943 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -62,7 +62,7 @@ authorized release action. | overlays | `@bsv/overlay-discovery-services` | `2.2.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | | overlays | `@bsv/overlay-express` | `2.6.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | | overlays | `@bsv/overlay-topics` | `1.7.1` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | -| sdk | `@bsv/sdk` | `2.4.2` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | +| sdk | `@bsv/sdk` | `2.5.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | | sdk | `@bsv/verifast` | `0.3.5` | wasm-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global, wasm-worker | browser, node, umd, wasm, worker | `>=22` | [packages/verifast](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/verifast) | | wallet | `@bsv/btms` | `1.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | | wallet | `@bsv/btms-permission-module` | `1.1.4` | node-library | node-esm | node | `>=22` | [packages/wallet/btms-permission-module](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms-permission-module) | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index eccf304bb..148d9582c 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -167,9 +167,9 @@ { "name": "@bsv/sdk", "publishedVersion": "2.4.0", - "releaseType": "patch", - "summary": "Adds one strict BRC-100 byte-boundary contract and secure React Native bridge validation, and completes incorporated-code compliance with hash-pinned provenance, exact MIT/ISC/BSD/Apache notices, source lineage markers, package payloads, a retained UMD banner, SBOM components, and release enforcement. Removes a broken copied Swagger UI scaffold and screenshot; the first-party OpenAPI source remains available for use with any viewer. Standardizes first-party author metadata on the current BSV Association name.", - "migration": "No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles." + "releaseType": "minor", + "summary": "Adds one strict BRC-100 byte-boundary contract and secure React Native bridge validation, and completes incorporated-code compliance with hash-pinned provenance, exact MIT/ISC/BSD/Apache notices, source lineage markers, package payloads, a retained UMD banner, SBOM components, and release enforcement. Removes a broken copied Swagger UI scaffold and screenshot; the first-party OpenAPI source remains available for use with any viewer. Standardizes first-party author metadata on the current BSV Association name. Adds a generic bounded transaction-evidence coordinator with derived-txid work sharing, alternate proof candidates, exact script-work reuse, canonical dependency rechecks, context fencing and reference-counted cancellation. Adds optional pre-aggregation lookup evidence receipts while preserving legacy resolver defaults. Adds optional fresh canonical-context tokens, with explicit limits for legacy tracker and remote ABA assurance.", + "migration": "No API migration is required. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. Existing Transaction.verify and resolver call shapes remain unchanged. New coordinator callers supply an independently maintained ChainTracker, explicit chain namespace and semantic policy id, handle bounded limit/timeout outcomes, and dispose sessions when finished. Optional ChainTracker context markers and cancellation signals are additive. No wire or storage migration is required. Coordinate resolver evidenceLimits with downstream evidence admission budgets when increasing them; limit events mean partial bounded intake, not invalid transaction evidence." }, { "name": "@bsv/simple", @@ -217,22 +217,22 @@ "name": "@bsv/wallet-toolbox", "publishedVersion": "2.10.4", "releaseType": "minor", - "summary": "Adds the built-in BRC-177 noSend-expiry reference implementation with exact prefunding, durable pre-signed reclaim, atomic active-storage monitoring, backoff-controlled recovery, cross-device lifecycle synchronization, and proof-finalized race handling. Adds opt-in prepared BEEF storage for Knex-backed normal createAction funding: verified, checksummed proof closures are persisted after foreground completion and reused on later hits, while broad lookups, misses, and cache failures retain the canonical path. Reads, writes, bounded queueing, and gradual backfill default off; reorganizations stale derived rows and fence in-flight cross-process writes with a database proof epoch. Also adds the optional semantic handleRequest hook for BRC-98/99/111 permission modules, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes, retains BRC-95/BRC-100 compatibility and stable bounded pagination, removes the obsolete JSight application bundle, and preserves the earlier Open BSV grant. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use.", - "migration": "Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness." + "summary": "Adds the built-in BRC-177 noSend-expiry reference implementation with exact prefunding, durable pre-signed reclaim, atomic active-storage monitoring, backoff-controlled recovery, cross-device lifecycle synchronization, and proof-finalized race handling. Adds opt-in prepared BEEF storage for Knex-backed normal createAction funding: verified, checksummed proof closures are persisted after foreground completion and reused on later hits, while broad lookups, misses, and cache failures retain the canonical path. Reads, writes, bounded queueing, and gradual backfill default off; reorganizations stale derived rows and fence in-flight cross-process writes with a database proof epoch. Also adds the optional semantic handleRequest hook for BRC-98/99/111 permission modules, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes, retains BRC-95/BRC-100 compatibility and stable bounded pagination, removes the obsolete JSight application bundle, and preserves the earlier Open BSV grant. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. Shares bounded transaction and identity cryptographic work across final discovery calls, retains alternate receipts before resolver deduplication, rechecks current canonical anchors, and fixes stale Chaintracks height-root reuse with provider/mode context fencing.", + "migration": "Existing actions, ordinary noSend calls, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate every active Knex store before serving requests and run the default Wallet Toolbox monitor; IndexedDB upgrades automatically to schema version 5. Upgrade signer, active storage service, and remote monitor together to 2.11.0 or later; older remote storage is rejected before prefunding. The Knex migration also adds rebuildable prepared-BEEF and proof-epoch tables with every COOK control disabled. Validate the migration on MySQL before release and the cross-process epoch fence on non-production PXC before enabling writes. Roll out writes before reads, use backfill only after database review, and disable all three flags to roll back. Delete derived prepared rows before downgrading to code that cannot advance the epoch. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Canonical AtomicBEEF and number-array behavior are unchanged; use @bsv/sdk 2.5.0 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. Identity RPC, trust filtering, local contacts and stored data remain unchanged; no additional migration is needed. The unpublished 2.11.1 candidate aggregates C01/C02/C03 changes against published 2.10.4. New evidence limits are local admission policy, not consensus limits; applications using the generic coordinator can configure them explicitly. Identity lookup propagates typed limit/timeout failures instead of returning a definitive empty result when bounded work cannot complete. Direct utility callers can pass matching queryOverlayEvidence and IdentityEvidenceVerifier byte limits. ChainTracks adapters bracket validation with fresh tips and local reorg/reset epochs where supported; remote observations are not atomic snapshots." }, { "name": "@bsv/wallet-toolbox-client", "publishedVersion": "2.10.4", "releaseType": "minor", - "summary": "Adds the built-in browser BRC-177 noSend-expiry signer, IndexedDB schema version 5 lifecycle state, remote storage capability negotiation, default monitor coordination, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while IndexedDB remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, stable IndexedDB totals, current browser compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use.", - "migration": "Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness." + "summary": "Adds the built-in browser BRC-177 noSend-expiry signer, IndexedDB schema version 5 lifecycle state, remote storage capability negotiation, default monitor coordination, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback when WebAssembly is unavailable, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while IndexedDB remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, stable IndexedDB totals, current browser compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. Shares bounded transaction and identity cryptographic work across final discovery calls, retains alternate receipts before resolver deduplication, rechecks current canonical anchors, and fixes stale Chaintracks height-root reuse with provider/mode context fencing.", + "migration": "Existing browser actions, permission modules, and UMP v3 tokens require no migration; IndexedDB upgrades automatically. To use BRC-177 with remote storage, upgrade the active storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so IndexedDB and remote clients require no COOK configuration. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes. Host registration is available from each package root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. Browser exports, wire types, canonical AtomicBEEF behavior, and pagination contracts are unchanged; use @bsv/sdk 2.5.0 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. Identity RPC, trust filtering, local contacts and stored data remain unchanged; no additional migration is needed. The unpublished 2.11.1 candidate aggregates C01/C02/C03 changes against published 2.10.4. New evidence limits are local admission policy, not consensus limits; applications using the generic coordinator can configure them explicitly. Identity lookup propagates typed limit/timeout failures instead of returning a definitive empty result when bounded work cannot complete. Direct utility callers can pass matching queryOverlayEvidence and IdentityEvidenceVerifier byte limits. ChainTracks adapters bracket validation with fresh tips and local reorg/reset epochs where supported; remote observations are not atomic snapshots." }, { "name": "@bsv/wallet-toolbox-mobile", "publishedVersion": "2.10.4", "releaseType": "minor", - "summary": "Adds the built-in mobile BRC-177 noSend-expiry signer, remote storage capability negotiation, default-monitor ownership coordination across restarts and devices, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback for React Native runtimes without WebAssembly, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while mobile storage remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, current mobile compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use.", - "migration": "Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness." + "summary": "Adds the built-in mobile BRC-177 noSend-expiry signer, remote storage capability negotiation, default-monitor ownership coordination across restarts and devices, the optional semantic handleRequest hook, an interoperable asynchronous JavaScript Argon2id fallback for React Native runtimes without WebAssembly, and an optional proven-ready native Argon2id backend for host runtimes. Carries the shared prepared-BEEF-aware createAction contract while mobile storage remains canonical-only; compatible remote Knex providers can enable COOK independently. Retains transformation modules, BRC-100 wire compatibility, current mobile compatibility fixes, and earlier Open BSV grants. Secures final overlay identity discovery with independently verified, byte-bound transaction evidence, subject-signed output fields, and explicit certificate success; cached response evidence is revalidated before use. Shares bounded transaction and identity cryptographic work across final discovery calls, retains alternate receipts before resolver deduplication, rechecks current canonical anchors, and fixes stale Chaintracks height-root reuse with provider/mode context fencing.", + "migration": "Existing mobile actions, permission modules, and UMP v3 tokens require no migration. To use BRC-177, migrate and upgrade the active remote storage service and its default monitor to Wallet Toolbox 2.11.0 or later before upgrading clients; an older server is rejected before prefunding. Prepared BEEF persistence and rollout controls apply only to the full package's Knex provider, so mobile remote clients require no COOK configuration. Semantic modules may add handleRequest without changing the Wallet interface. Argon2id tokens keep the same parameters and derived bytes across WebAssembly and JavaScript runtimes; no user device setting is required. Host registration is available from the mobile root; concurrent cold calls share a preload attempt, and hosts must make readiness/preload reentrant and cache permanent failures or back off retries. Native and JavaScript results share byte-type and exact-length validation; unrelated hash-wasm errors still propagate without WebAssembly. React Native exports, wire types, and canonical AtomicBEEF behavior are unchanged; use @bsv/sdk 2.5.0 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/. Identity discovery requires the wallet Services ChainTracker to provide canonical chain evidence. No wallet RPC or storage migration is required for this security correction. Direct identityUtils callers must pass their independently maintained ChainTracker; missing verification context and unverifiable overlay candidates no longer produce identities. Local contacts retain separate provenance; inclusion is not proof of freshness or unspentness. Identity RPC, trust filtering, local contacts and stored data remain unchanged; no additional migration is needed. The unpublished 2.11.1 candidate aggregates C01/C02/C03 changes against published 2.10.4. New evidence limits are local admission policy, not consensus limits; applications using the generic coordinator can configure them explicitly. Identity lookup propagates typed limit/timeout failures instead of returning a definitive empty result when bounded work cannot complete. Direct utility callers can pass matching queryOverlayEvidence and IdentityEvidenceVerifier byte limits. ChainTracks adapters bracket validation with fresh tips and local reorg/reset epochs where supported; remote observations are not atomic snapshots." }, { "name": "create-bsv-app", diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index c75e4504c..17ed9eb42 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -322,7 +322,7 @@ "@bsv/overlay-discovery-services": "2.2.1", "@bsv/overlay-express": "2.6.1", "@bsv/overlay-topics": "1.7.1", - "@bsv/sdk": "2.4.2", + "@bsv/sdk": "2.5.0", "@bsv/verifast": "0.3.5", "@bsv/btms": "1.2.2", "@bsv/btms-permission-module": "1.1.4", diff --git a/packages/sdk/README.md b/packages/sdk/README.md index ee0e7d30e..42b97ef5a 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -16,6 +16,11 @@ emits a portable `number[]` settlement artifact so HTTP, WebSocket, Message Box, and JSON transports preserve identical transaction bytes. The same boundary protects overlay lookup queries and JSON BEEF responses. +Security-sensitive overlay consumers can use the exported bounded transaction +evidence coordinator to verify copied BEEF against a caller-owned chain tracker. +See the [transaction evidence guide](./docs/transaction-evidence.md) for the +receipt, graph, cache, cancellation, and resolver-intake contracts. + ## Table of Contents 1. [Objective](#objective) @@ -118,6 +123,7 @@ Comprehensive documentation is available in several formats: - **[🔧 How-To Guides](https://bsv-blockchain.github.io/ts-stack/guides/)**: Practical solutions to specific problems - **[📚 Reference](https://bsv-blockchain.github.io/ts-stack/reference/)**: Complete technical specifications and API documentation - **[🏗️ Architecture](https://bsv-blockchain.github.io/ts-stack/architecture/)**: Architecture and design explanations +- **[Transaction evidence](./docs/transaction-evidence.md)**: Bounded verification of untrusted BEEF and overlay receipts - **[⚡ Examples](https://docs.bsvblockchain.org/guides/sdks/ts/examples)**: Practical code examples - **Code Annotations**: The SDK is richly documented with code-level annotations that show up in editors like VSCode diff --git a/packages/sdk/docs/reference/overlay-tools.md b/packages/sdk/docs/reference/overlay-tools.md index ec167e2bb..2ba53ad88 100644 --- a/packages/sdk/docs/reference/overlay-tools.md +++ b/packages/sdk/docs/reference/overlay-tools.md @@ -4,16 +4,15 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ## Interfaces -| | -| --- | -| [AdmittanceInstructions](#interface-admittanceinstructions) | -| [LookupQuestion](#interface-lookupquestion) | -| [LookupResolverConfig](#interface-lookupresolverconfig) | -| [OverlayBroadcastFacilitator](#interface-overlaybroadcastfacilitator) | -| [OverlayLookupFacilitator](#interface-overlaylookupfacilitator) | -| [RankedHost](#interface-rankedhost) | -| [SHIPBroadcasterConfig](#interface-shipbroadcasterconfig) | -| [TaggedBEEF](#interface-taggedbeef) | +| | | +| ----------------------------------------------------------- | --------------------------------------------------------------------- | +| [AdmittanceInstructions](#interface-admittanceinstructions) | [OverlayBroadcastFacilitator](#interface-overlaybroadcastfacilitator) | +| [LookupAnswerProgress](#interface-lookupanswerprogress) | [OverlayLookupFacilitator](#interface-overlaylookupfacilitator) | +| [LookupFreeformAnswer](#interface-lookupfreeformanswer) | [RankedHost](#interface-rankedhost) | +| [LookupQueryOptions](#interface-lookupqueryoptions) | [SHIPBroadcasterConfig](#interface-shipbroadcasterconfig) | +| [LookupQuestion](#interface-lookupquestion) | [TaggedBEEF](#interface-taggedbeef) | +| [LookupResolution](#interface-lookupresolution) | [UnreachableHostInfo](#interface-unreachablehostinfo) | +| [LookupResolverConfig](#interface-lookupresolverconfig) | | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -25,9 +24,9 @@ Instructs the Overlay Services Engine about which outputs to admit and which pre ```ts export interface AdmittanceInstructions { - outputsToAdmit: number[]; - coinsToRetain: number[]; - coinsRemoved?: number[]; + outputsToAdmit: number[] + coinsToRetain: number[] + coinsRemoved?: number[] } ``` @@ -59,14 +58,264 @@ outputsToAdmit: number[] Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Interface: LookupAnswerProgress + +```ts +export interface LookupAnswerProgress { + type: 'output-list' + outputs: Array<{ + beef: number[] + outputIndex: number + context?: number[] + txid?: string + }> + txIds: string[] + isFinal: boolean + hostCount: number + completedHosts: number + successfulHosts: number + emptyHosts: number + failedHosts: number + rejectedHosts: number + freeformHosts: number + correlationId?: string +} +``` + +#### Property completedHosts + +Number of hosts that have settled (success / fail / timeout). + +```ts +completedHosts: number +``` + +#### Property correlationId + +Correlation id used for privacy-safe distributed diagnostics. + +```ts +correlationId?: string +``` + +#### Property emptyHosts + +Successful hosts whose output list was empty. + +```ts +emptyHosts: number +``` + +#### Property failedHosts + +Hosts that failed due to availability, timeout, or malformed responses. + +```ts +failedHosts: number +``` + +#### Property freeformHosts + +Hosts that returned a valid but non-aggregatable freeform response. + +```ts +freeformHosts: number +``` + +#### Property hostCount + +Number of ranked hosts that were queried. + +```ts +hostCount: number +``` + +#### Property isFinal + +True only for the final emission, after every in-flight host has settled. + +```ts +isFinal: boolean +``` + +#### Property rejectedHosts + +Hosts that rejected this query semantically (for example, HTTP 400). + +```ts +rejectedHosts: number +``` + +#### Property successfulHosts + +Hosts that returned a structurally valid output-list response. + +```ts +successfulHosts: number +``` + +#### Property txIds + +Parallel array of resolved tx ids for each output (same index as `outputs`). + +```ts +txIds: string[] +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Interface: LookupFreeformAnswer + +A valid non-aggregatable response returned by a lookup service. + +```ts +export interface LookupFreeformAnswer { + type: 'freeform' + result: unknown +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Interface: LookupQueryOptions + +```ts +export interface LookupQueryOptions { + onEvidence?: (event: LookupEvidenceEvent) => void | Promise + evidenceLimits?: { + maxOutputs?: number + maxBytes?: number + } + graceMs?: number + softTimeoutMs?: number + onUnreachableHost?: (info: UnreachableHostInfo) => void | Promise + unreachableHostNotificationCooldownMs?: number + holdForUnknownHosts?: boolean + waitForAllHosts?: boolean + correlationId?: string +} +``` + +See also: [LookupEvidenceEvent](./overlay-tools.md#type-lookupevidenceevent), [UnreachableHostInfo](./overlay-tools.md#interface-unreachablehostinfo) + +#### Property correlationId + +Correlates resolver and downstream wallet telemetry without logging the query payload. + +```ts +correlationId?: string +``` + +#### Property evidenceLimits + +Callback intake budget, independent of legacy aggregation. Defaults to 512 +outputs / 16 MiB of BEEF and context bytes. Values must be positive safe +integers. Coordinate these with a downstream verifier's admission limits. + +```ts +evidenceLimits?: { + maxOutputs?: number; + maxBytes?: number; +} +``` + +#### Property graceMs + +Override the grace window (ms) between the first valid response and the resolution of the query. +Late responders arriving within this window are merged into the result. Default 80 ms. +Raise for identity-style paths (e.g. ~300 ms) where divergence between hosts matters. + +```ts +graceMs?: number +``` + +#### Property holdForUnknownHosts + +Compatibility alias for `waitForAllHosts`. Prefer `waitForAllHosts` in new +code. `waitForAllHosts` takes precedence when both are supplied. + +```ts +holdForUnknownHosts?: boolean +``` + +#### Property onEvidence + +Owned, UNTRUSTED receipts before legacy txid/outpoint deduplication. Enqueue +promptly; callback completion is not awaited and failures are isolated. +Intake stops at the configured evidenceLimits, reporting one limit event. +No callbacks occur after the query iterator closes. Legacy answers, host +scheduling, timeout and reputation behavior are unchanged. + +```ts +onEvidence?: (event: LookupEvidenceEvent) => void | Promise +``` + +See also: [LookupEvidenceEvent](./overlay-tools.md#type-lookupevidenceevent) + +#### Property onUnreachableHost + +Fired when a SLAP-advertised host fails (network error, timeout, malformed +response). The resolver itself does not email or escalate — downstream +consumers (e.g. overlay-express) wire this up to the BSVA notification API +to let the originating overlay operator know about a stale advertisement. + +```ts +onUnreachableHost?: (info: UnreachableHostInfo) => void | Promise +``` + +See also: [UnreachableHostInfo](./overlay-tools.md#interface-unreachablehostinfo) + +#### Property softTimeoutMs + +Soft timeout (ms). When set: + +- `query()` resolves with whatever has arrived as soon as any host answers, or after this timeout. +- `query$()` emits a (possibly empty) snapshot after this timeout if no host has answered yet, + then continues yielding late-host enrichments until the iterator is broken or final emission. + +```ts +softTimeoutMs?: number +``` + +#### Property unreachableHostNotificationCooldownMs + +Minimum interval between unreachable notifications for the same host and +service. Defaults to 60 seconds to prevent notification storms. Set to 0 +to disable deduplication. + +```ts +unreachableHostNotificationCooldownMs?: number +``` + +#### Property waitForAllHosts + +Wait for every queried host to settle before the first emission. This is +the default for `query()` because generic output cardinality is not proof +of freshness or authority. It defaults to `false` for progressive +`query$()` consumers. `holdForUnknownHosts` remains as a compatibility +alias; `waitForAllHosts` takes precedence when both are supplied. + +```ts +waitForAllHosts?: boolean +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Interface: LookupQuestion The question asked to the Overlay Services Engine when a consumer of state wishes to look up information. ```ts export interface LookupQuestion { - service: string; - query: unknown; + service: string + query: unknown } ``` @@ -90,26 +339,47 @@ service: string Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Interface: LookupResolution + +A lookup answer together with the host settlement evidence behind it. + +```ts +export interface LookupResolution { + answer: LookupAnswer + progress: LookupAnswerProgress +} +``` + +See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupAnswerProgress](./overlay-tools.md#interface-lookupanswerprogress) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Interface: LookupResolverConfig Configuration options for the Lookup resolver. ```ts export interface LookupResolverConfig { - networkPreset?: "mainnet" | "testnet" | "teratestnet" | "local"; - facilitator?: OverlayLookupFacilitator; - slapTrackers?: string[]; - hostOverrides?: Record; - additionalHosts?: Record; - cache?: CacheOptions; - reputationStorage?: "localStorage" | { - get: (key: string) => string | null | undefined; - set: (key: string, value: string) => void; - }; + networkPreset?: LookupNetworkPreset + facilitator?: OverlayLookupFacilitator + slapTrackers?: string[] + hostOverrides?: Record + additionalHosts?: Record + cache?: CacheOptions + reputationStorage?: + | 'localStorage' + | { + get: (key: string) => string | null | undefined + set: (key: string, value: string) => void + } + telemetry?: TelemetryConfig } ``` -See also: [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) +See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset), [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) #### Property additionalHosts @@ -134,6 +404,7 @@ The facilitator used to make requests to Overlay Services hosts. ```ts facilitator?: OverlayLookupFacilitator ``` + See also: [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) #### Property hostOverrides @@ -147,14 +418,18 @@ hostOverrides?: Record #### Property networkPreset The network preset to use, unless other options override it. + - mainnet: use mainnet SLAP trackers and HTTPS facilitator - testnet: use testnet SLAP trackers and HTTPS facilitator +- teratestnet: use TerraTestNet SLAP trackers and HTTPS facilitator - local: directly query from localhost:8080 and a facilitator that permits plain HTTP ```ts -networkPreset?: "mainnet" | "testnet" | "teratestnet" | "local" +networkPreset?: LookupNetworkPreset ``` +See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset) + #### Property reputationStorage Optional storage for host reputation data. @@ -174,16 +449,25 @@ The list of SLAP trackers queried to resolve Overlay Services hosts for a given slapTrackers?: string[] ``` +#### Property telemetry + +Optional privacy-bounded telemetry sink. Query payloads are never emitted. + +```ts +telemetry?: TelemetryConfig +``` + Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: OverlayBroadcastFacilitator Facilitates transaction broadcasts that return STEAK. ```ts export interface OverlayBroadcastFacilitator { - send: (url: string, taggedBEEF: TaggedBEEF) => Promise; + send: (url: string, taggedBEEF: TaggedBEEF) => Promise } ``` @@ -192,57 +476,66 @@ See also: [STEAK](./overlay-tools.md#type-steak), [TaggedBEEF](./overlay-tools.m Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: OverlayLookupFacilitator Facilitates lookups to URLs that return answers. ```ts export interface OverlayLookupFacilitator { - lookup: (url: string, question: LookupQuestion, timeout?: number) => Promise; + lookup: ( + url: string, + question: LookupQuestion, + timeout?: number + ) => Promise } ``` -See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) +See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) #### Property lookup Returns a lookup answer for a lookup question ```ts -lookup: (url: string, question: LookupQuestion, timeout?: number) => Promise +lookup: (url: string, question: LookupQuestion, timeout?: number) => + Promise ``` -See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) + +See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: RankedHost ```ts export interface RankedHost extends HostReputationEntry { - score: number; + score: number } ``` Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: SHIPBroadcasterConfig Configuration options for the SHIP broadcaster. ```ts export interface SHIPBroadcasterConfig { - networkPreset?: "mainnet" | "testnet" | "teratestnet" | "local"; - facilitator?: OverlayBroadcastFacilitator; - resolver?: LookupResolver; - requireAcknowledgmentFromAllHostsForTopics?: "all" | "any" | string[]; - requireAcknowledgmentFromAnyHostForTopics?: "all" | "any" | string[]; - requireAcknowledgmentFromSpecificHostsForTopics?: Record; + networkPreset?: LookupNetworkPreset + facilitator?: OverlayBroadcastFacilitator + resolver?: LookupResolver + requireAcknowledgmentFromAllHostsForTopics?: TopicAcknowledgmentRequirement + requireAcknowledgmentFromAnyHostForTopics?: TopicAcknowledgmentRequirement + requireAcknowledgmentFromSpecificHostsForTopics?: Record } ``` -See also: [LookupResolver](./overlay-tools.md#class-lookupresolver), [OverlayBroadcastFacilitator](./overlay-tools.md#interface-overlaybroadcastfacilitator) +See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset), [LookupResolver](./overlay-tools.md#class-lookupresolver), [OverlayBroadcastFacilitator](./overlay-tools.md#interface-overlaybroadcastfacilitator), [TopicAcknowledgmentRequirement](./overlay-tools.md#type-topicacknowledgmentrequirement) #### Property facilitator @@ -251,43 +544,54 @@ The facilitator used to make requests to Overlay Services hosts. ```ts facilitator?: OverlayBroadcastFacilitator ``` + See also: [OverlayBroadcastFacilitator](./overlay-tools.md#interface-overlaybroadcastfacilitator) #### Property networkPreset The network preset to use, unless other options override it. + - mainnet: use mainnet resolver and HTTPS facilitator - testnet: use testnet resolver and HTTPS facilitator +- teratestnet: use TerraTestNet resolver and HTTPS facilitator - local: directly send to localhost:8080 and a facilitator that permits plain HTTP ```ts -networkPreset?: "mainnet" | "testnet" | "teratestnet" | "local" +networkPreset?: LookupNetworkPreset ``` +See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset) + #### Property requireAcknowledgmentFromAllHostsForTopics Determines which topics (all, any, or a specific list) must be present within all STEAKs received from every host for the broadcast to be considered a success. By default, all hosts must acknowledge all topics. ```ts -requireAcknowledgmentFromAllHostsForTopics?: "all" | "any" | string[] +requireAcknowledgmentFromAllHostsForTopics?: TopicAcknowledgmentRequirement ``` +See also: [TopicAcknowledgmentRequirement](./overlay-tools.md#type-topicacknowledgmentrequirement) + #### Property requireAcknowledgmentFromAnyHostForTopics Determines which topics (all, any, or a specific list) must be present within STEAK received from at least one host for the broadcast to be considered a success. ```ts -requireAcknowledgmentFromAnyHostForTopics?: "all" | "any" | string[] +requireAcknowledgmentFromAnyHostForTopics?: TopicAcknowledgmentRequirement ``` +See also: [TopicAcknowledgmentRequirement](./overlay-tools.md#type-topicacknowledgmentrequirement) + #### Property requireAcknowledgmentFromSpecificHostsForTopics Determines a mapping whose keys are specific hosts and whose values are the topics (all, any, or a specific list) that must be present within the STEAK received by the given hosts, in order for the broadcast to be considered a success. ```ts -requireAcknowledgmentFromSpecificHostsForTopics?: Record +requireAcknowledgmentFromSpecificHostsForTopics?: Record ``` +See also: [TopicAcknowledgmentRequirement](./overlay-tools.md#type-topicacknowledgmentrequirement) + #### Property resolver The resolver used to locate suitable hosts with SHIP @@ -295,36 +599,89 @@ The resolver used to locate suitable hosts with SHIP ```ts resolver?: LookupResolver ``` + See also: [LookupResolver](./overlay-tools.md#class-lookupresolver) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: TaggedBEEF Tagged BEEF ```ts export interface TaggedBEEF { - beef: number[]; - topics: string[]; - offChainValues?: number[]; + beef: number[] + topics: string[] + offChainValues?: number[] +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Interface: UnreachableHostInfo + +Info supplied to onUnreachableHost callbacks. + +```ts +export interface UnreachableHostInfo { + host: string + service: string + error: string + advertisedBy?: string } ``` +#### Property advertisedBy + +SLAP tracker URL that advertised this host, if known. + +```ts +advertisedBy?: string +``` + +#### Property error + +Error message from the facilitator. + +```ts +error: string +``` + +#### Property host + +Host URL that failed. + +```ts +host: string +``` + +#### Property service + +Lookup service that was being queried when the failure occurred. + +```ts +service: string +``` + Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ## Classes -| | -| --- | +| | +| --------------------------------------------------------------------------- | | [HTTPSOverlayBroadcastFacilitator](#class-httpsoverlaybroadcastfacilitator) | -| [HTTPSOverlayLookupFacilitator](#class-httpsoverlaylookupfacilitator) | -| [HostReputationTracker](#class-hostreputationtracker) | -| [LookupResolver](#class-lookupresolver) | -| [OverlayAdminTokenTemplate](#class-overlayadmintokentemplate) | -| [TopicBroadcaster](#class-topicbroadcaster) | +| [HTTPSOverlayLookupFacilitator](#class-httpsoverlaylookupfacilitator) | +| [HostReputationTracker](#class-hostreputationtracker) | +| [LookupHTTPError](#class-lookuphttperror) | +| [LookupResolver](#class-lookupresolver) | +| [OverlayAdminTokenTemplate](#class-overlayadmintokentemplate) | +| [TopicBroadcaster](#class-topicbroadcaster) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -334,10 +691,10 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ```ts export class HTTPSOverlayBroadcastFacilitator implements OverlayBroadcastFacilitator { - httpClient: typeof fetch; - allowHTTP: boolean; - constructor(httpClient = fetch, allowHTTP: boolean = false) - async send(url: string, taggedBEEF: TaggedBEEF): Promise + httpClient: typeof fetch + allowHTTP: boolean + constructor(httpClient = fetch, allowHTTP: boolean = false) + async send(url: string, taggedBEEF: TaggedBEEF): Promise } ``` @@ -346,84 +703,154 @@ See also: [OverlayBroadcastFacilitator](./overlay-tools.md#interface-overlaybroa Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: HTTPSOverlayLookupFacilitator ```ts export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { - fetchClient: typeof fetch; - allowHTTP: boolean; - constructor(httpClient = defaultFetch, allowHTTP: boolean = false) - async lookup(url: string, question: LookupQuestion, timeout: number = 5000): Promise + fetchClient: typeof fetch + allowHTTP: boolean + constructor(httpClient = defaultFetch, allowHTTP: boolean = false) + async lookup( + url: string, + question: LookupQuestion, + timeout: number = 2000 + ): Promise } ``` -See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) +See also: [LookupFacilitatorAnswer](./overlay-tools.md#type-lookupfacilitatoranswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [OverlayLookupFacilitator](./overlay-tools.md#interface-overlaylookupfacilitator) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: HostReputationTracker ```ts export class HostReputationTracker { - constructor(store?: KeyValueStore) - reset(): void - recordSuccess(host: string, latencyMs: number): void - recordFailure(host: string, reason?: unknown): void - rankHosts(hosts: string[], now: number = Date.now()): RankedHost[] - snapshot(host: string): HostReputationEntry | undefined + constructor(store?: KeyValueStore) + reset(): void + recordSuccess(host: string, latencyMs: number): void + recordFailure(host: string, reason?: unknown): void + rankHosts(hosts: string[], now: number = Date.now()): RankedHost[] + snapshot(host: string): HostReputationEntry | undefined + flush(): void } ``` See also: [RankedHost](./overlay-tools.md#interface-rankedhost) +#### Method flush + +Flushes a pending debounced persistence write immediately. + +```ts +flush(): void +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Class: LookupHTTPError + +An HTTP failure with enough classification for reputation handling. + +```ts +export class LookupHTTPError extends Error { + readonly status: number + readonly kind: LookupHTTPErrorKind + constructor(status: number, kind: LookupHTTPErrorKind, statusText?: string) +} +``` + +See also: [LookupHTTPErrorKind](./overlay-tools.md#type-lookuphttperrorkind) + Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: LookupResolver Represents a Lookup Resolver. ```ts export default class LookupResolver { - constructor(config: LookupResolverConfig = {}) - async query(question: LookupQuestion, timeout?: number): Promise + constructor(config: LookupResolverConfig = {}) + async query( + question: LookupQuestion, + timeout?: number, + options?: LookupQueryOptions + ): Promise + async queryDetailed( + question: LookupQuestion, + timeout?: number, + options?: LookupQueryOptions + ): Promise + async *query$( + question: LookupQuestion, + timeout?: number, + options?: LookupQueryOptions + ): AsyncIterable } ``` -See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupResolverConfig](./overlay-tools.md#interface-lookupresolverconfig) +See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupAnswerProgress](./overlay-tools.md#interface-lookupanswerprogress), [LookupQueryOptions](./overlay-tools.md#interface-lookupqueryoptions), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupResolution](./overlay-tools.md#interface-lookupresolution), [LookupResolverConfig](./overlay-tools.md#interface-lookupresolverconfig) #### Method query Given a LookupQuestion, returns a LookupAnswer. Aggregates across multiple services and supports resiliency. +Optional `options.graceMs` overrides the per-call grace window (default 80 ms). +Optional `options.softTimeoutMs` resolves the query early with whatever has arrived once any host has +answered (or with an empty result if no host has answered by `softTimeoutMs`). + ```ts -async query(question: LookupQuestion, timeout?: number): Promise +async query(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise ``` -See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) + +See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupQueryOptions](./overlay-tools.md#interface-lookupqueryoptions), [LookupQuestion](./overlay-tools.md#interface-lookupquestion) + +#### Method queryDetailed + +Performs a lookup and returns both its answer and the host settlement +evidence required by security-sensitive consumers to distinguish an +authoritative empty result from an availability failure. + +```ts +async queryDetailed(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise +``` + +See also: [LookupQueryOptions](./overlay-tools.md#interface-lookupqueryoptions), [LookupQuestion](./overlay-tools.md#interface-lookupquestion), [LookupResolution](./overlay-tools.md#interface-lookupresolution) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: OverlayAdminTokenTemplate Script template enabling the creation, unlocking, and decoding of SHIP and SLAP advertisements. ```ts export default class OverlayAdminTokenTemplate implements ScriptTemplate { - pushDrop: PushDrop; - static decode(script: LockingScript): { - protocol: "SHIP" | "SLAP"; - identityKey: string; - domain: string; - topicOrService: string; - } - constructor(wallet: WalletInterface, originator?: OriginatorDomainNameStringUnder250Bytes) - async lock(protocol: "SHIP" | "SLAP", domain: string, topicOrService: string): Promise - unlock(protocol: "SHIP" | "SLAP"): { - sign: (tx: Transaction, inputIndex: number) => Promise; - estimateLength: (tx: Transaction, inputIndex: number) => Promise; - } + pushDrop: PushDrop + static decode(script: LockingScript): { + protocol: 'SHIP' | 'SLAP' + identityKey: string + domain: string + topicOrService: string + } + constructor(wallet: WalletInterface, originator?: OriginatorDomainNameStringUnder250Bytes) + async lock( + protocol: 'SHIP' | 'SLAP', + domain: string, + topicOrService: string + ): Promise + unlock(protocol: 'SHIP' | 'SLAP'): { + sign: (tx: Transaction, inputIndex: number) => Promise + estimateLength: (tx: Transaction, inputIndex: number) => Promise + } } ``` @@ -434,14 +861,15 @@ See also: [LockingScript](./script.md#class-lockingscript), [OriginatorDomainNam Constructs a new Overlay Admin template instance ```ts -constructor(wallet: WalletInterface, originator?: OriginatorDomainNameStringUnder250Bytes) +constructor(wallet: WalletInterface, originator?: OriginatorDomainNameStringUnder250Bytes) ``` + See also: [OriginatorDomainNameStringUnder250Bytes](./wallet.md#type-originatordomainnamestringunder250bytes), [WalletInterface](./wallet.md#interface-walletinterface) Argument Details -+ **wallet** - + Wallet to use for locking and unlocking +- **wallet** + - Wallet to use for locking and unlocking #### Method decode @@ -453,8 +881,9 @@ static decode(script: LockingScript): { identityKey: string; domain: string; topicOrService: string; -} +} ``` + See also: [LockingScript](./script.md#class-lockingscript) Returns @@ -463,16 +892,17 @@ Decoded SHIP or SLAP advertisement Argument Details -+ **script** - + Locking script comprising a SHIP or SLAP token to decode +- **script** + - Locking script comprising a SHIP or SLAP token to decode #### Method lock Creates a new advertisement locking script ```ts -async lock(protocol: "SHIP" | "SLAP", domain: string, topicOrService: string): Promise +async lock(protocol: "SHIP" | "SLAP", domain: string, topicOrService: string): Promise ``` + See also: [LockingScript](./script.md#class-lockingscript) Returns @@ -481,12 +911,12 @@ Locking script comprising the advertisement token Argument Details -+ **protocol** - + SHIP or SLAP -+ **domain** - + Domain where the topic or service is available -+ **topicOrService** - + Topic or service to advertise +- **protocol** + - SHIP or SLAP +- **domain** + - Domain where the topic or service is available +- **topicOrService** + - Topic or service to advertise #### Method unlock @@ -496,8 +926,9 @@ Unlocks an advertisement token as part of a transaction. unlock(protocol: "SHIP" | "SLAP"): { sign: (tx: Transaction, inputIndex: number) => Promise; estimateLength: (tx: Transaction, inputIndex: number) => Promise; -} +} ``` + See also: [Transaction](./transaction.md#class-transaction), [UnlockingScript](./script.md#class-unlockingscript), [sign](./compat.md#variable-sign) Returns @@ -506,20 +937,21 @@ Script unlocker capable of unlocking the advertisement token Argument Details -+ **protocol** - + SHIP or SLAP, depending on the token to unlock +- **protocol** + - SHIP or SLAP, depending on the token to unlock Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: TopicBroadcaster Broadcasts transactions to one or more overlay topics. ```ts export default class TopicBroadcaster implements Broadcaster { - constructor(topics: string[], config: SHIPBroadcasterConfig = {}) - async broadcast(tx: Transaction): Promise + constructor(topics: string[], config: SHIPBroadcasterConfig = {}) + async broadcast(tx: Transaction): Promise } ``` @@ -530,24 +962,26 @@ See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [Broa Constructs an instance of the SHIP broadcaster. ```ts -constructor(topics: string[], config: SHIPBroadcasterConfig = {}) +constructor(topics: string[], config: SHIPBroadcasterConfig = {}) ``` + See also: [SHIPBroadcasterConfig](./overlay-tools.md#interface-shipbroadcasterconfig) Argument Details -+ **topics** - + The list of SHIP topic names where transactions are to be sent. -+ **config** - + Configuration options for the SHIP broadcaster. +- **topics** + - The list of SHIP topic names where transactions are to be sent. +- **config** + - Configuration options for the SHIP broadcaster. #### Method broadcast Broadcasts a transaction to Overlay Services via SHIP. ```ts -async broadcast(tx: Transaction): Promise +async broadcast(tx: Transaction): Promise ``` + See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [BroadcastResponse](./transaction.md#interface-broadcastresponse), [Transaction](./transaction.md#class-transaction) Returns @@ -556,12 +990,13 @@ A promise that resolves to either a success or failure response. Argument Details -+ **tx** - + The transaction to be sent. +- **tx** + - The transaction to be sent. Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ## Functions ### Function: withDoubleSpendRetry @@ -571,7 +1006,11 @@ When a double-spend is detected, broadcasts the competing transaction to update the overlay with missing state, then retries the operation. ```ts -export async function withDoubleSpendRetry(operation: () => Promise, broadcaster: TopicBroadcaster, maxRetries: number = MAX_DOUBLE_SPEND_RETRIES): Promise +export async function withDoubleSpendRetry( + operation: () => Promise, + broadcaster: TopicBroadcaster, + maxRetries: number = MAX_DOUBLE_SPEND_RETRIES +): Promise ``` See also: [TopicBroadcaster](./overlay-tools.md#class-topicbroadcaster) @@ -582,12 +1021,12 @@ The result of the successful operation Argument Details -+ **operation** - + The async operation to execute (e.g., createAction + signAction) -+ **broadcaster** - + The TopicBroadcaster to use for syncing missing state -+ **maxRetries** - + Maximum number of retry attempts (default: MAX_DOUBLE_SPEND_RETRIES) +- **operation** + - The async operation to execute (e.g., createAction + signAction) +- **broadcaster** + - The TopicBroadcaster to use for syncing missing state +- **maxRetries** + - Maximum number of retry attempts (default: MAX_DOUBLE_SPEND_RETRIES) Throws @@ -596,12 +1035,19 @@ If max retries exceeded or non-double-spend error occurs Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ## Types -| | -| --- | -| [LookupAnswer](#type-lookupanswer) | -| [STEAK](#type-steak) | +| | +| ---------------------------------------------------------------------- | +| [LookupAnswer](#type-lookupanswer) | +| [LookupEvidenceEvent](#type-lookupevidenceevent) | +| [LookupFacilitatorAnswer](#type-lookupfacilitatoranswer) | +| [LookupHTTPErrorKind](#type-lookuphttperrorkind) | +| [LookupNetworkPreset](#type-lookupnetworkpreset) | +| [RequireMode](#type-requiremode) | +| [STEAK](#type-steak) | +| [TopicAcknowledgmentRequirement](#type-topicacknowledgmentrequirement) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -609,23 +1055,94 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Type: LookupAnswer -How the Overlay Services Engine responds to a Lookup Question. -It may comprise either an output list or a freeform response from the Lookup Service. +An aggregatable output-list answer returned by the resolver. ```ts export type LookupAnswer = { - type: "output-list"; - outputs: Array<{ - beef: number[]; - outputIndex: number; - context?: number[]; - }>; + type: 'output-list' + outputs: Array<{ + beef: number[] + outputIndex: number + context?: number[] + txid?: string + }> } ``` Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Type: LookupEvidenceEvent + +Additive evidence intake, independent of the legacy aggregated answer. + +```ts +export type LookupEvidenceEvent = + | { + type: 'output' + host: string + output: LookupAnswer['outputs'][number] + } + | { + type: 'limit' + } +``` + +See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Type: LookupFacilitatorAnswer + +Responses a facilitator may return before the resolver aggregates them. + +```ts +export type LookupFacilitatorAnswer = LookupAnswer | LookupFreeformAnswer +``` + +See also: [LookupAnswer](./overlay-tools.md#type-lookupanswer), [LookupFreeformAnswer](./overlay-tools.md#interface-lookupfreeformanswer) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Type: LookupHTTPErrorKind + +```ts +export type LookupHTTPErrorKind = 'semantic' | 'availability' +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Type: LookupNetworkPreset + +Public overlay network presets understood by lookup and SHIP routing. + +```ts +export type LookupNetworkPreset = 'mainnet' | 'testnet' | 'teratestnet' | 'local' +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Type: RequireMode + +The require mode for topic acknowledgment: all topics must be present, or any one suffices. + +```ts +export type RequireMode = 'all' | 'any' +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Type: STEAK Submitted Transaction Execution AcKnowledgment @@ -639,14 +1156,30 @@ See also: [AdmittanceInstructions](./overlay-tools.md#interface-admittanceinstru Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Type: TopicAcknowledgmentRequirement + +Specifies which topics must be acknowledged: all, any, or a specific list. + +```ts +export type TopicAcknowledgmentRequirement = RequireMode | string[] +``` + +See also: [RequireMode](./overlay-tools.md#type-requiremode) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ## Enums ## Variables -| | -| --- | -| [DEFAULT_SLAP_TRACKERS](#variable-default_slap_trackers) | -| [DEFAULT_TESTNET_SLAP_TRACKERS](#variable-default_testnet_slap_trackers) | +| | +| ---------------------------------------------------------------------------- | +| [DEFAULT_SLAP_TRACKERS](#variable-default_slap_trackers) | +| [DEFAULT_TESTNET_SLAP_TRACKERS](#variable-default_testnet_slap_trackers) | +| [DEFAULT_TTN_SLAP_TRACKERS](#variable-default_ttn_slap_trackers) | | [getOverlayHostReputationTracker](#variable-getoverlayhostreputationtracker) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -667,6 +1200,7 @@ DEFAULT_SLAP_TRACKERS: string[] = [ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Variable: DEFAULT_TESTNET_SLAP_TRACKERS ```ts @@ -678,6 +1212,19 @@ DEFAULT_TESTNET_SLAP_TRACKERS: string[] = [ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Variable: DEFAULT_TTN_SLAP_TRACKERS + +```ts +DEFAULT_TTN_SLAP_TRACKERS: string[] = [ + "https://staging-overlay.babbage.systems" +] +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Variable: getOverlayHostReputationTracker ```ts diff --git a/packages/sdk/docs/reference/transaction.md b/packages/sdk/docs/reference/transaction.md index d1791a8b2..7d74d8b00 100644 --- a/packages/sdk/docs/reference/transaction.md +++ b/packages/sdk/docs/reference/transaction.md @@ -4,17 +4,17 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ## Interfaces -| | | -| --- | --- | -| [ArcConfig](#interface-arcconfig) | [HttpsModuleLike](#interface-httpsmodulelike) | -| [BroadcastFailure](#interface-broadcastfailure) | [HttpsNodejs](#interface-httpsnodejs) | -| [BroadcastResponse](#interface-broadcastresponse) | [MerklePathLeaf](#interface-merklepathleaf) | -| [Broadcaster](#interface-broadcaster) | [NodejsHttpClientRequest](#interface-nodejshttpclientrequest) | -| [ChainTracker](#interface-chaintracker) | [NodejsRequestLike](#interface-nodejsrequestlike) | -| [FeeModel](#interface-feemodel) | [TransactionInput](#interface-transactioninput) | -| [FetchOptions](#interface-fetchoptions) | [TransactionOutput](#interface-transactionoutput) | -| [HttpClient](#interface-httpclient) | [WhatsOnChainConfig](#interface-whatsonchainconfig) | -| [HttpClientRequestOptions](#interface-httpclientrequestoptions) | | +| | | | +| ----------------------------------------------------------- | --------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | +| [ArcConfig](#interface-arcconfig) | [FeeModel](#interface-feemodel) | [TransactionEvidence](#interface-transactionevidence) | +| [BdkVerifierInterface](#interface-bdkverifierinterface) | [FetchOptions](#interface-fetchoptions) | [TransactionEvidenceContext](#interface-transactionevidencecontext) | +| [BdkVerifyScriptsParams](#interface-bdkverifyscriptsparams) | [HttpClient](#interface-httpclient) | [TransactionEvidenceCoordinatorOptions](#interface-transactionevidencecoordinatoroptions) | +| [BroadcastFailure](#interface-broadcastfailure) | [HttpClientRequestOptions](#interface-httpclientrequestoptions) | [TransactionEvidenceLimits](#interface-transactionevidencelimits) | +| [BroadcastResponse](#interface-broadcastresponse) | [HttpsModuleLike](#interface-httpsmodulelike) | [TransactionInput](#interface-transactioninput) | +| [Broadcaster](#interface-broadcaster) | [HttpsNodejs](#interface-httpsnodejs) | [TransactionOutput](#interface-transactionoutput) | +| [ChainTracker](#interface-chaintracker) | [MerklePathLeaf](#interface-merklepathleaf) | [VerifiedTransactionOutput](#interface-verifiedtransactionoutput) | +| [EvidenceCandidate](#interface-evidencecandidate) | [NodejsHttpClientRequest](#interface-nodejshttpclientrequest) | [WhatsOnChainConfig](#interface-whatsonchainconfig) | +| [EvidenceScriptScope](#interface-evidencescriptscope) | [NodejsRequestLike](#interface-nodejsrequestlike) | | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -26,12 +26,12 @@ Configuration options for the ARC broadcaster. ```ts export interface ArcConfig { - apiKey?: string; - httpClient?: HttpClient; - deploymentId?: string; - callbackUrl?: string; - callbackToken?: string; - headers?: Record; + apiKey?: string + httpClient?: HttpClient + deploymentId?: string + callbackUrl?: string + callbackToken?: string + headers?: Record } ``` @@ -84,44 +84,132 @@ The HTTP client used to make requests to the ARC API. ```ts httpClient?: HttpClient ``` + See also: [HttpClient](./transaction.md#interface-httpclient) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Interface: BdkVerifierInterface + +A pluggable backend that verifies ALL input scripts of a single transaction. + +Implementations (e.g. + +```ts +export default interface BdkVerifierInterface { + supportsMemoryLimit?: boolean + shouldVerifyScripts?: (params: BdkVerifyScriptsParams) => boolean + verifyScripts: (params: BdkVerifyScriptsParams) => Promise + verifyScriptsBatch?: (params: readonly BdkVerifyScriptsParams[]) => Promise +} +``` + +See also: [BdkVerifyScriptsParams](./transaction.md#interface-bdkverifyscriptsparams) + +#### Property shouldVerifyScripts + +Optionally decide whether this backend should handle the transaction now. +Returning false preserves the SDK's synchronous JavaScript interpreter path. +Implementations can use this to avoid waiting for a cold optional backend. + +```ts +shouldVerifyScripts?: (params: BdkVerifyScriptsParams) => boolean +``` + +See also: [BdkVerifyScriptsParams](./transaction.md#interface-bdkverifyscriptsparams) + +#### Property supportsMemoryLimit + +True only when this backend applies `params.memoryLimit` during script +execution. Backends that omit this capability are bypassed for calls with +an explicit memory limit. + +```ts +supportsMemoryLimit?: boolean +``` + +#### Property verifyScripts + +Verify all input scripts of `params.tx`. + +```ts +verifyScripts: (params: BdkVerifyScriptsParams) => Promise +``` + +See also: [BdkVerifyScriptsParams](./transaction.md#interface-bdkverifyscriptsparams) + +#### Property verifyScriptsBatch + +Verify several independent transactions in one backend scheduling pass. +Implementations may use packed native calls and worker-level parallelism. + +```ts +verifyScriptsBatch?: (params: readonly BdkVerifyScriptsParams[]) => Promise +``` + +See also: [BdkVerifyScriptsParams](./transaction.md#interface-bdkverifyscriptsparams) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Interface: BdkVerifyScriptsParams + +Parameters shared by script-verifier routing and execution. + +```ts +export interface BdkVerifyScriptsParams { + tx: Transaction + blockHeight: number + consensus: boolean + verifyFlags?: string | string[] + memoryLimit?: number +} +``` + +See also: [Transaction](./transaction.md#class-transaction) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Interface: BroadcastFailure Defines the structure of a failed broadcast response. ```ts export interface BroadcastFailure { - status: "error"; - code: string; - txid?: string; - description: string; - more?: object; + status: 'error' + code: string + txid?: string + description: string + more?: object } ``` Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: BroadcastResponse Defines the structure of a successful broadcast response. ```ts export interface BroadcastResponse { - status: "success"; - txid: string; - message: string; - competingTxs?: string[]; + status: 'success' + txid: string + message: string + competingTxs?: string[] } ``` Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: Broadcaster Represents the interface for a transaction broadcaster. @@ -129,8 +217,8 @@ This interface defines a standard method for broadcasting transactions. ```ts export interface Broadcaster { - broadcast: (transaction: Transaction) => Promise; - broadcastMany?: (txs: Transaction[]) => Promise; + broadcast: (transaction: Transaction) => Promise + broadcastMany?: (txs: Transaction[]) => Promise } ``` @@ -139,6 +227,7 @@ See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [Broa Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: ChainTracker The Chain Tracker is responsible for verifying the validity of a given Merkle root @@ -163,14 +252,77 @@ const chainTracker = { ```ts export default interface ChainTracker { - isValidRootForHeight: (root: string, height: number) => Promise; - currentHeight: () => Promise; + isValidRootForHeight: (root: string, height: number, signal?: AbortSignal) => Promise + currentHeight: (signal?: AbortSignal) => Promise + getVerificationContext?: () => string | number + getVerificationContextToken?: (signal?: AbortSignal) => Promise } ``` +#### Property getVerificationContext + +Optional trusted local provider/policy/recovery context. Change this value +when switching sources or resetting their state. It is not a canonical +chain snapshot: consumers must still check current canonical dependencies. +Implementations without cancellable I/O may ignore the optional signals. + +```ts +getVerificationContext?: () => string | number +``` + +#### Property getVerificationContextToken + +Optional fresh canonical context token from the trusted chain provider. +Include canonical block/tip identity and any available monotonic reorg or +reset epoch. Consumers compare tokens around asynchronous verification. +Two remote tip observations are not an atomic snapshot and cannot detect +an intervening transition back to the identical tip (ABA). + +```ts +getVerificationContextToken?: (signal?: AbortSignal) => Promise +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Interface: EvidenceCandidate + +Internal owned candidate. Never constructed from a host's verification assertion. + +```ts +export interface EvidenceCandidate { + tx: Transaction + txid: string + receipt: string + byteLength: number + outputIndex: number + graphBinding: string +} +``` + +See also: [Transaction](./transaction.md#class-transaction) + Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Interface: EvidenceScriptScope + +```ts +export interface EvidenceScriptScope { + work: EvidenceScriptWork + signal: AbortSignal + check: () => void +} +``` + +See also: [EvidenceScriptWork](./transaction.md#class-evidencescriptwork) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Interface: FeeModel Represents the interface for a transaction fee model. @@ -178,7 +330,7 @@ This interface defines a standard method for computing a fee when given a transa ```ts export default interface FeeModel { - computeFee: (transaction: Transaction) => Promise; + computeFee: (transaction: Transaction) => Promise } ``` @@ -187,6 +339,7 @@ See also: [Transaction](./transaction.md#class-transaction) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: FetchOptions An interface for configuration of the request to be passed to the fetch method @@ -194,9 +347,9 @@ limited to options needed by ts-sdk. ```ts export interface FetchOptions { - method?: string; - headers?: Record; - body?: string | null; + method?: string + headers?: Record + body?: string | null } ``` @@ -227,13 +380,17 @@ method?: string Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: HttpClient An interface for HTTP client used to make HTTP requests. ```ts export interface HttpClient { - request: (url: string, options: HttpClientRequestOptions) => Promise>; + request: ( + url: string, + options: HttpClientRequestOptions + ) => Promise> } ``` @@ -244,23 +401,26 @@ See also: [HttpClientRequestOptions](./transaction.md#interface-httpclientreques Makes a request to the server. ```ts -request: (url: string, options: HttpClientRequestOptions) => Promise> +request: (url: string, options: HttpClientRequestOptions) => + Promise> ``` + See also: [HttpClientRequestOptions](./transaction.md#interface-httpclientrequestoptions), [HttpClientResponse](./transaction.md#type-httpclientresponse) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: HttpClientRequestOptions An interface for configuration of the request to be passed to the request method. ```ts export interface HttpClientRequestOptions { - method?: string; - headers?: Record; - data?: Data; - signal?: AbortSignal; + method?: string + headers?: Record + data?: Data + signal?: AbortSignal } ``` @@ -299,13 +459,18 @@ signal?: AbortSignal Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: HttpsModuleLike Common interface for Node.js https modules ```ts export interface HttpsModuleLike { - request: (url: string, options: HttpClientRequestOptions, callback: (res: any) => void) => NodejsRequestLike; + request: ( + url: string, + options: HttpClientRequestOptions, + callback: (res: any) => void + ) => NodejsRequestLike } ``` @@ -314,13 +479,18 @@ See also: [HttpClientRequestOptions](./transaction.md#interface-httpclientreques Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: HttpsNodejs Node Https module interface limited to options needed by ts-sdk ```ts export interface HttpsNodejs { - request: (url: string, options: HttpClientRequestOptions, callback: (res: any) => void) => NodejsHttpClientRequest; + request: ( + url: string, + options: HttpClientRequestOptions, + callback: (res: any) => void + ) => NodejsHttpClientRequest } ``` @@ -329,50 +499,147 @@ See also: [HttpClientRequestOptions](./transaction.md#interface-httpclientreques Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: MerklePathLeaf ```ts export interface MerklePathLeaf { - offset: number; - hash?: string; - txid?: boolean; - duplicate?: boolean; + offset: number + hash?: string + txid?: boolean + duplicate?: boolean } ``` Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: NodejsHttpClientRequest Nodejs result of the Node https.request call limited to options needed by ts-sdk ```ts export interface NodejsHttpClientRequest { - write: (chunk: string) => void; - on: (event: string, callback: (data: any) => void) => void; - end: () => void; + write: (chunk: string) => void + on: (event: string, callback: (data: any) => void) => void + end: () => void } ``` Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: NodejsRequestLike Common interface for Node.js https module request objects ```ts export interface NodejsRequestLike { - write: (chunk: any) => void; - on: (event: string, callback: (data: any) => void) => void; - end: () => void; + write: (chunk: any) => void + on: (event: string, callback: (data: any) => void) => void + end: () => void +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Interface: TransactionEvidence + +Untrusted transaction evidence. The optional txid is only a consistency hint. + +```ts +export interface TransactionEvidence { + beef: number[] + outputIndex: number + txid?: string +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Interface: TransactionEvidenceContext + +Caller-controlled trust configuration; never populate this from lookup metadata. + +```ts +export interface TransactionEvidenceContext { + chainTracker: ChainTracker + chainNamespace: string + policyId: string + verifier?: BdkVerifierInterface +} +``` + +See also: [BdkVerifierInterface](./transaction.md#interface-bdkverifierinterface), [ChainTracker](./transaction.md#interface-chaintracker) + +#### Property chainNamespace + +Network/genesis identifier or an explicit application chain namespace. + +```ts +chainNamespace: string +``` + +#### Property policyId + +Semantic verification policy/backend version; change it when policy changes. + +```ts +policyId: string +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Interface: TransactionEvidenceCoordinatorOptions + +```ts +export interface TransactionEvidenceCoordinatorOptions extends TransactionEvidenceContext { + limits?: Partial } ``` +See also: [TransactionEvidenceContext](./transaction.md#interface-transactionevidencecontext), [TransactionEvidenceLimits](./transaction.md#interface-transactionevidencelimits) + Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Interface: TransactionEvidenceLimits + +Local admission policy, not consensus limits. Byte limits count serialized bytes. + +```ts +export interface TransactionEvidenceLimits { + candidateBytes: number + retainedBytes: number + transactions: number + inputs: number + scriptBytes: number + scriptMemoryBytes: number + candidatesPerTransaction: number + pendingTransactions: number + concurrentTransactions: number + pendingChainCalls: number + consumers: number + cacheEntries: number + cacheAgeMs: number + attemptTimeoutMs: number + requestTimeoutMs: number +} +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Interface: TransactionInput Represents an input to a Bitcoin transaction. @@ -398,15 +665,15 @@ txInput.unlockingScriptTemplate = { ```ts export default interface TransactionInput { - sourceTransaction?: Transaction; - sourceTXID?: string; - sourceOutputIndex: number; - unlockingScript?: UnlockingScript; - unlockingScriptTemplate?: { - sign: (tx: Transaction, inputIndex: number) => Promise; - estimateLength: (tx: Transaction, inputIndex: number) => Promise; - }; - sequence?: number; + sourceTransaction?: Transaction + sourceTXID?: string + sourceOutputIndex: number + unlockingScript?: UnlockingScript + unlockingScriptTemplate?: { + sign: (tx: Transaction, inputIndex: number) => Promise + estimateLength: (tx: Transaction, inputIndex: number) => Promise + } + sequence?: number } ``` @@ -415,6 +682,7 @@ See also: [Transaction](./transaction.md#class-transaction), [UnlockingScript](. Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Interface: TransactionOutput Represents an output in a Bitcoin transaction. @@ -429,14 +697,14 @@ let txOutput = { satoshis: 1000, lockingScript: LockingScript.fromASM('OP_DUP OP_HASH160 ... OP_EQUALVERIFY OP_CHECKSIG'), change: false -}; +} ``` ```ts export default interface TransactionOutput { - satoshis?: number; - lockingScript: LockingScript; - change?: boolean; + satoshis?: number + lockingScript: LockingScript + change?: boolean } ``` @@ -445,14 +713,34 @@ See also: [LockingScript](./script.md#class-lockingscript) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Interface: VerifiedTransactionOutput + +Verified transaction inclusion/ancestry; no service relevance or unspentness claim. + +```ts +export interface VerifiedTransactionOutput { + readonly txid: string + readonly outputIndex: number + readonly outpoint: string + readonly lockingScript: LockingScript +} +``` + +See also: [LockingScript](./script.md#class-lockingscript) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Interface: WhatsOnChainConfig Configuration options for the WhatsOnChain ChainTracker. ```ts export interface WhatsOnChainConfig { - apiKey?: string; - httpClient?: HttpClient; + apiKey?: string + httpClient?: HttpClient } ``` @@ -473,26 +761,24 @@ The HTTP client used to make requests to the API. ```ts httpClient?: HttpClient ``` + See also: [HttpClient](./transaction.md#interface-httpclient) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ## Classes -| | -| --- | -| [ARC](#class-arc) | -| [Beef](#class-beef) | -| [BeefParty](#class-beefparty) | -| [BeefTx](#class-beeftx) | -| [FetchHttpClient](#class-fetchhttpclient) | -| [LivePolicy](#class-livepolicy) | -| [MerklePath](#class-merklepath) | -| [NodejsHttpClient](#class-nodejshttpclient) | -| [SatoshisPerKilobyte](#class-satoshisperkilobyte) | -| [Transaction](#class-transaction) | -| [WhatsOnChain](#class-whatsonchain) | +| | | +| ----------------------------------------------- | ----------------------------------------------------------------------- | +| [ARC](#class-arc) | [MerklePath](#class-merklepath) | +| [Beef](#class-beef) | [NodejsHttpClient](#class-nodejshttpclient) | +| [BeefParty](#class-beefparty) | [SatoshisPerKilobyte](#class-satoshisperkilobyte) | +| [BeefTx](#class-beeftx) | [Transaction](#class-transaction) | +| [EvidenceScriptWork](#class-evidencescriptwork) | [TransactionEvidenceCoordinator](#class-transactionevidencecoordinator) | +| [FetchHttpClient](#class-fetchhttpclient) | [TransactionEvidenceError](#class-transactionevidenceerror) | +| [LivePolicy](#class-livepolicy) | [WhatsOnChain](#class-whatsonchain) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -504,17 +790,17 @@ Represents an ARC transaction broadcaster. ```ts export default class ARC implements Broadcaster { - readonly URL: string; - readonly apiKey: string | undefined; - readonly deploymentId: string; - readonly callbackUrl: string | undefined; - readonly callbackToken: string | undefined; - readonly headers: Record | undefined; - constructor(URL: string, config?: ArcConfig); - constructor(URL: string, apiKey?: string); - constructor(URL: string, config?: string | ArcConfig) - async broadcast(tx: Transaction): Promise - async broadcastMany(txs: Transaction[]): Promise + readonly URL: string + readonly apiKey: string | undefined + readonly deploymentId: string + readonly callbackUrl: string | undefined + readonly callbackToken: string | undefined + readonly headers: Record | undefined + constructor(URL: string, config?: ArcConfig) + constructor(URL: string, apiKey?: string) + constructor(URL: string, config?: string | ArcConfig) + async broadcast(tx: Transaction): Promise + async broadcastMany(txs: Transaction[]): Promise } ``` @@ -527,14 +813,15 @@ Constructs an instance of the ARC broadcaster. ```ts constructor(URL: string, config?: ArcConfig) ``` + See also: [ArcConfig](./transaction.md#interface-arcconfig) Argument Details -+ **URL** - + The URL endpoint for the ARC API. -+ **config** - + Configuration options for the ARC broadcaster. +- **URL** + - The URL endpoint for the ARC API. +- **config** + - Configuration options for the ARC broadcaster. #### Constructor @@ -546,18 +833,19 @@ constructor(URL: string, apiKey?: string) Argument Details -+ **URL** - + The URL endpoint for the ARC API. -+ **apiKey** - + The API key used for authorization with the ARC API. +- **URL** + - The URL endpoint for the ARC API. +- **apiKey** + - The API key used for authorization with the ARC API. #### Method broadcast Broadcasts a transaction via ARC. ```ts -async broadcast(tx: Transaction): Promise +async broadcast(tx: Transaction): Promise ``` + See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [BroadcastResponse](./transaction.md#interface-broadcastresponse), [Transaction](./transaction.md#class-transaction) Returns @@ -566,8 +854,8 @@ A promise that resolves to either a success or failure response. Argument Details -+ **tx** - + The transaction to be broadcasted. +- **tx** + - The transaction to be broadcasted. #### Method broadcastMany @@ -575,8 +863,9 @@ Broadcasts multiple transactions via ARC. Handles mixed responses where some transactions succeed and others fail. ```ts -async broadcastMany(txs: Transaction[]): Promise +async broadcastMany(txs: Transaction[]): Promise ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -585,60 +874,70 @@ A promise that resolves to an array of objects. Argument Details -+ **txs** - + Array of transactions to be broadcasted. +- **txs** + - Array of transactions to be broadcasted. Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: Beef ```ts export class Beef { - bumps: MerklePath[] = []; - txs: BeefTx[] = []; - version: number = BEEF_V2; - atomicTxid: string | undefined = undefined; - constructor(version: number = BEEF_V2) - findTxid(txid: string): BeefTx | undefined - makeTxidOnly(txid: string): BeefTx | undefined - findBump(txid: string): MerklePath | undefined - findTransactionForSigning(txid: string): Transaction | undefined - findAtomicTransaction(txid: string): Transaction | undefined - mergeBump(bump: MerklePath): number - mergeRawTx(rawTx: number[] | Uint8Array, bumpIndex?: number): BeefTx - mergeTransaction(tx: Transaction): BeefTx - removeExistingTxid(txid: string): void - mergeTxidOnly(txid: string): BeefTx - mergeBeefTx(btx: BeefTx): BeefTx - mergeBeef(beef: Beef | number[] | Uint8Array): void - isValid(allowTxidOnly?: boolean): boolean - async verify(chainTracker: ChainTracker, allowTxidOnly?: boolean): Promise - verifyValid(allowTxidOnly?: boolean): { - valid: boolean; - roots: Record; - } - toWriter(writer: Writer | WriterUint8Array): void - toBinary(): number[] - toUint8Array(): Uint8Array - toBinaryAtomic(txid: string): number[] - toUint8ArrayAtomic(txid: string): Uint8Array - toHex(): string - static fromReader(br: Reader | ReaderUint8Array): Beef - static fromBinary(bin: number[] | Uint8Array): Beef - static fromString(s: string, enc: "hex" | "utf8" | "base64" = "hex"): Beef - sortTxs(): { - missingInputs: string[]; - notValid: string[]; - valid: string[]; - withMissingInputs: string[]; - txidOnly: string[]; - } - clone(): Beef - trimKnownTxids(knownTxids: string[]): void - getValidTxids(): string[] - toLogString(): string - addComputedLeaves(): void + bumps: MerklePath[] = [] + txs: BeefTx[] = [] + version: number = BEEF_V2 + atomicTxid: string | undefined = undefined + constructor(version: number = BEEF_V2) + isAtomic(txid: string = this.atomicTxid ?? ''): boolean + findTxid(txid: string): BeefTx | undefined + makeTxidOnly(txid: string): BeefTx | undefined + findBump(txid: string): MerklePath | undefined + findTransactionForSigning(txid: string): Transaction | undefined + findAtomicTransaction(txid: string): Transaction | undefined + mergeBump(bump: MerklePath): number + mergeProvenTxs( + entries: Array<{ + rawTx: number[] | Uint8Array + merklePath: MerklePath + merkleRoot?: string + }> + ): BeefTx[] + mergeRawTx(rawTx: number[] | Uint8Array, bumpIndex?: number): BeefTx + mergeTransaction(tx: Transaction): BeefTx + removeExistingTxid(txid: string): void + mergeTxidOnly(txid: string): BeefTx + mergeBeefTx(btx: BeefTx): BeefTx + mergeBeef(beef: Beef | number[] | Uint8Array): void + isValid(allowTxidOnly?: boolean): boolean + async verify(chainTracker: ChainTracker, allowTxidOnly?: boolean): Promise + verifyValid(allowTxidOnly?: boolean): { + valid: boolean + roots: Record + } + toWriter(writer: Writer | WriterUint8Array): void + toBinary(): number[] + toUint8Array(): Uint8Array + toBinaryAtomic(txid: string): number[] + toUint8ArrayAtomic(txid: string): Uint8Array + toHex(): string + static fromReader(br: Reader | ReaderUint8Array): Beef + static fromBinary(bin: number[] | Uint8Array): Beef + static fromBinaryView(bin: Uint8Array): Beef + static fromString(s: string, enc: 'hex' | 'utf8' | 'base64' = 'hex'): Beef + sortTxs(): { + missingInputs: string[] + notValid: string[] + valid: string[] + withMissingInputs: string[] + txidOnly: string[] + } + clone(): Beef + trimKnownTxids(knownTxids: string[]): void + getValidTxids(): string[] + toLogString(): string + addComputedLeaves(): void } ``` @@ -650,14 +949,15 @@ In some circumstances it may be helpful for the BUMP MerklePaths to include leaves that can be computed from row zero. ```ts -addComputedLeaves(): void +addComputedLeaves(): void ``` #### Method clone ```ts -clone(): Beef +clone(): Beef ``` + See also: [Beef](./transaction.md#class-beef) Returns @@ -671,8 +971,9 @@ Builds the proof tree rooted at a specific `Transaction`. To succeed, the Beef must contain all the required transaction and merkle path data. ```ts -findAtomicTransaction(txid: string): Transaction | undefined +findAtomicTransaction(txid: string): Transaction | undefined ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -681,14 +982,15 @@ Transaction with input `SourceTransaction` and `MerklePath` populated from this Argument Details -+ **txid** - + The id of the target transaction. +- **txid** + - The id of the target transaction. #### Method findBump ```ts -findBump(txid: string): MerklePath | undefined +findBump(txid: string): MerklePath | undefined ``` + See also: [MerklePath](./transaction.md#class-merklepath) Returns @@ -703,8 +1005,9 @@ and adds any missing input SourceTransactions from this `Beef`. The result is suitable for signing. ```ts -findTransactionForSigning(txid: string): Transaction | undefined +findTransactionForSigning(txid: string): Transaction | undefined ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -713,14 +1016,15 @@ Transaction with all available input `SourceTransaction`s from this Beef. Argument Details -+ **txid** - + The id of the target transaction. +- **txid** + - The id of the target transaction. #### Method findTxid ```ts -findTxid(txid: string): BeefTx | undefined +findTxid(txid: string): BeefTx | undefined ``` + See also: [BeefTx](./transaction.md#class-beeftx) Returns @@ -729,16 +1033,17 @@ Returns Argument Details -+ **txid** - + of `beefTx` to find +- **txid** + - of `beefTx` to find #### Method fromBinary Constructs an instance of the Beef class based on the provided binary array ```ts -static fromBinary(bin: number[] | Uint8Array): Beef +static fromBinary(bin: number[] | Uint8Array): Beef ``` + See also: [Beef](./transaction.md#class-beef) Returns @@ -747,16 +1052,28 @@ An instance of the Beef class constructed from the binary data Argument Details -+ **bin** - + The binary array or Uint8Array from which to construct BEEF +- **bin** + - The binary array or Uint8Array from which to construct BEEF + +#### Method fromBinaryView + +Parses BEEF while retaining zero-copy views over `bin`. The caller must not +mutate the buffer for the lifetime of the returned object. + +```ts +static fromBinaryView(bin: Uint8Array): Beef +``` + +See also: [Beef](./transaction.md#class-beef) #### Method fromString Constructs an instance of the Beef class based on the provided string ```ts -static fromString(s: string, enc: "hex" | "utf8" | "base64" = "hex"): Beef +static fromString(s: string, enc: "hex" | "utf8" | "base64" = "hex"): Beef ``` + See also: [Beef](./transaction.md#class-beef) Returns @@ -765,21 +1082,31 @@ An instance of the Beef class constructed from the string Argument Details -+ **s** - + The string value from which to construct BEEF -+ **enc** - + The encoding of the string value from which BEEF should be constructed +- **s** + - The string value from which to construct BEEF +- **enc** + - The encoding of the string value from which BEEF should be constructed #### Method getValidTxids ```ts -getValidTxids(): string[] +getValidTxids(): string[] ``` Returns array of transaction txids that either have a proof or whose inputs chain back to a proven transaction. +#### Method isAtomic + +Checks the BRC-95 transaction-inclusion rule without requiring header-root +validation: the subject must exist and every included transaction must be +in its recursive dependency graph. + +```ts +isAtomic(txid: string = this.atomicTxid ?? ""): boolean +``` + #### Method isValid Sorts `txs` and checks structural validity of beef. @@ -787,19 +1114,20 @@ Sorts `txs` and checks structural validity of beef. Does NOT verify merkle roots. Validity requirements: + 1. No 'known' txids, unless `allowTxidOnly` is true. 2. All transactions have bumps or their inputs chain back to bumps (or are known). 3. Order of transactions satisfies dependencies before dependents. 4. No transactions with duplicate txids. ```ts -isValid(allowTxidOnly?: boolean): boolean +isValid(allowTxidOnly?: boolean): boolean ``` Argument Details -+ **allowTxidOnly** - + optional. If true, transaction txid only is assumed valid +- **allowTxidOnly** + - optional. If true, transaction txid only is assumed valid #### Method makeTxidOnly @@ -810,8 +1138,9 @@ updated by this method without affecting the original. ```ts -makeTxidOnly(txid: string): BeefTx | undefined +makeTxidOnly(txid: string): BeefTx | undefined ``` + See also: [BeefTx](./transaction.md#class-beeftx) Returns @@ -823,14 +1152,34 @@ undefined if txid is unknown. Merge a MerklePath that is assumed to be fully valid. ```ts -mergeBump(bump: MerklePath): number +mergeBump(bump: MerklePath): number ``` + See also: [MerklePath](./transaction.md#class-merklepath) Returns index of merged bump +#### Method mergeProvenTxs + +Merge several independently proven transactions in one mutation pass. + +This is equivalent to calling `mergeRawTx` followed by `mergeBump` for +every entry, but synchronizes nested BEEF state only once. That distinction +matters for wallets assembling a BEEF from a fragmented UTXO set because +proof paths are otherwise re-scanned after every input. + +```ts +mergeProvenTxs(entries: Array<{ + rawTx: number[] | Uint8Array; + merklePath: MerklePath; + merkleRoot?: string; +}>): BeefTx[] +``` + +See also: [BeefTx](./transaction.md#class-beeftx), [MerklePath](./transaction.md#class-merklepath) + #### Method mergeRawTx Merge a serialized transaction. @@ -840,8 +1189,9 @@ Checks that a transaction with the same txid hasn't already been merged. Replaces existing transaction with same txid. ```ts -mergeRawTx(rawTx: number[] | Uint8Array, bumpIndex?: number): BeefTx +mergeRawTx(rawTx: number[] | Uint8Array, bumpIndex?: number): BeefTx ``` + See also: [BeefTx](./transaction.md#class-beeftx) Returns @@ -850,8 +1200,8 @@ txid of rawTx Argument Details -+ **bumpIndex** - + Optional. If a number, must be valid index into bumps array. +- **bumpIndex** + - Optional. If a number, must be valid index into bumps array. #### Method mergeTransaction @@ -862,8 +1212,9 @@ Replaces existing transaction with same txid. Attempts to match an existing bump to the new transaction. ```ts -mergeTransaction(tx: Transaction): BeefTx +mergeTransaction(tx: Transaction): BeefTx ``` + See also: [BeefTx](./transaction.md#class-beeftx), [Transaction](./transaction.md#class-transaction) Returns @@ -875,17 +1226,18 @@ txid of tx Removes an existing transaction from the BEEF, given its TXID ```ts -removeExistingTxid(txid: string): void +removeExistingTxid(txid: string): void ``` Argument Details -+ **txid** - + TXID of the transaction to remove +- **txid** + - TXID of the transaction to remove #### Method sortTxs Sort the `txs` by input txid dependency order: + - Oldest Tx Anchored by Path or txid only - Newer Txs depending on Older parents - Newest Tx @@ -899,7 +1251,7 @@ sortTxs(): { valid: string[]; withMissingInputs: string[]; txidOnly: string[]; -} +} ``` Returns @@ -911,7 +1263,7 @@ Returns Returns a binary array representing the serialized BEEF ```ts -toBinary(): number[] +toBinary(): number[] ``` Returns @@ -926,10 +1278,10 @@ Serialize this Beef as AtomicBEEF. `txid` must exist -after sorting, if txid is not last txid, creates a clone and removes newer txs +includes exactly the subject transaction and its recursive dependencies ```ts -toBinaryAtomic(txid: string): number[] +toBinaryAtomic(txid: string): number[] ``` Returns @@ -941,7 +1293,7 @@ serialized contents of this Beef with AtomicBEEF prefix. Returns a hex string representing the serialized BEEF ```ts -toHex(): string +toHex(): string ``` Returns @@ -951,7 +1303,7 @@ A hex string representing the BEEF #### Method toLogString ```ts -toLogString(): string +toLogString(): string ``` Returns @@ -963,7 +1315,7 @@ Summary of `Beef` contents as multi-line string. Returns a binary array representing the serialized BEEF ```ts -toUint8Array(): Uint8Array +toUint8Array(): Uint8Array ``` Returns @@ -976,10 +1328,10 @@ Serialize this Beef as AtomicBEEF. `txid` must exist -after sorting, if txid is not last txid, creates a clone and removes newer txs +includes exactly the subject transaction and its recursive dependencies ```ts -toUint8ArrayAtomic(txid: string): Uint8Array +toUint8ArrayAtomic(txid: string): Uint8Array ``` Returns @@ -991,8 +1343,9 @@ serialized contents of this Beef with AtomicBEEF prefix. Serializes this data to `writer` ```ts -toWriter(writer: Writer | WriterUint8Array): void +toWriter(writer: Writer | WriterUint8Array): void ``` + See also: [Writer](./primitives.md#class-writer), [WriterUint8Array](./primitives.md#class-writeruint8array) #### Method trimKnownTxids @@ -1000,7 +1353,7 @@ See also: [Writer](./primitives.md#class-writer), [WriterUint8Array](./primitive Ensure that all the txids in `knownTxids` are txidOnly ```ts -trimKnownTxids(knownTxids: string[]): void +trimKnownTxids(knownTxids: string[]): void ``` #### Method verify @@ -1010,22 +1363,24 @@ by validating structure of this beef and confirming computed merkle roots using `chainTracker`. Validity requirements: + 1. No 'known' txids, unless `allowTxidOnly` is true. 2. All transactions have bumps or their inputs chain back to bumps (or are known). 3. Order of transactions satisfies dependencies before dependents. 4. No transactions with duplicate txids. ```ts -async verify(chainTracker: ChainTracker, allowTxidOnly?: boolean): Promise +async verify(chainTracker: ChainTracker, allowTxidOnly?: boolean): Promise ``` + See also: [ChainTracker](./transaction.md#interface-chaintracker) Argument Details -+ **chainTracker** - + Used to verify computed merkle path roots for all bump txids. -+ **allowTxidOnly** - + optional. If true, transaction txid is assumed valid +- **chainTracker** + - Used to verify computed merkle path roots for all bump txids. +- **allowTxidOnly** + - optional. If true, transaction txid is assumed valid #### Method verifyValid @@ -1035,6 +1390,7 @@ by validating structure of this beef. Returns block heights and merkle root values to be confirmed by a chaintracker. Validity requirements: + 1. No 'known' txids, unless `allowTxidOnly` is true. 2. All transactions have bumps or their inputs chain back to bumps (or are known). 3. Order of transactions satisfies dependencies before dependents. @@ -1044,7 +1400,7 @@ Validity requirements: verifyValid(allowTxidOnly?: boolean): { valid: boolean; roots: Record; -} +} ``` Returns @@ -1054,12 +1410,13 @@ Returns Argument Details -+ **allowTxidOnly** - + optional. If true, transaction txid is assumed valid +- **allowTxidOnly** + - optional. If true, transaction txid is assumed valid Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: BeefParty Extends `Beef` that is used to exchange transaction validity data with more than one external party. @@ -1083,14 +1440,17 @@ The size and redundancy of these Beefs becomes a problem when chained transactio ```ts export class BeefParty extends Beef { - knownTo: Record> = {}; - constructor(parties?: string[]) - isParty(party: string): boolean - addParty(party: string): void - getKnownTxidsForParty(party: string): string[] - getTrimmedBeefForParty(party: string): Beef - addKnownTxidsForParty(party: string, knownTxids: string[]): void - mergeBeefFromParty(party: string, beef: number[] | Beef): void + knownTo: Record> = Object.create(null) as Record< + string, + Record + > + constructor(parties?: string[]) + isParty(party: string): boolean + addParty(party: string): void + getKnownTxidsForParty(party: string): string[] + getTrimmedBeefForParty(party: string): Beef + addKnownTxidsForParty(party: string, knownTxids: string[]): void + mergeBeefFromParty(party: string, beef: number[] | Uint8Array | Beef): void } ``` @@ -1099,13 +1459,13 @@ See also: [Beef](./transaction.md#class-beef) #### Constructor ```ts -constructor(parties?: string[]) +constructor(parties?: string[]) ``` Argument Details -+ **parties** - + Optional array of initial unique party identifiers. +- **parties** + - Optional array of initial unique party identifiers. #### Property knownTo @@ -1113,7 +1473,10 @@ keys are party identifiers. values are records of txids with truthy value for which the party already has validity proof. ```ts -knownTo: Record> = {} +knownTo: Record> = Object.create(null) as Record< + string, + Record +> ``` #### Method addKnownTxidsForParty @@ -1121,26 +1484,26 @@ knownTo: Record> = {} Make note of additional txids "known" to `party`. ```ts -addKnownTxidsForParty(party: string, knownTxids: string[]): void +addKnownTxidsForParty(party: string, knownTxids: string[]): void ``` Argument Details -+ **party** - + unique identifier, added if new. +- **party** + - unique identifier, added if new. #### Method addParty Adds a new unique party identifier to this `BeefParty`. ```ts -addParty(party: string): void +addParty(party: string): void ``` #### Method getKnownTxidsForParty ```ts -getKnownTxidsForParty(party: string): string[] +getKnownTxidsForParty(party: string): string[] ``` Returns @@ -1150,8 +1513,9 @@ Array of txids "known" to `party`. #### Method getTrimmedBeefForParty ```ts -getTrimmedBeefForParty(party: string): Beef +getTrimmedBeefForParty(party: string): Beef ``` + See also: [Beef](./transaction.md#class-beef) Returns @@ -1161,7 +1525,7 @@ trimmed beef of unknown transactions and proofs for `party` #### Method isParty ```ts -isParty(party: string): boolean +isParty(party: string): boolean ``` Returns @@ -1177,13 +1541,15 @@ corresponding to transactions for which `party` has raw transaction and validity proof data. ```ts -mergeBeefFromParty(party: string, beef: number[] | Beef): void +mergeBeefFromParty(party: string, beef: number[] | Uint8Array | Beef): void ``` + See also: [Beef](./transaction.md#class-beef) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: BeefTx A single bitcoin transaction associated with a `Beef` validity proof set. @@ -1196,26 +1562,31 @@ which they can merge if necessary to create a valid beef. ```ts export default class BeefTx { - _bumpIndex?: number; - _tx?: Transaction; - _rawTx?: Uint8Array; - _txid?: string; - inputTxids: string[] = []; - isValid?: boolean = undefined; - get bumpIndex(): number | undefined - set bumpIndex(v: number | undefined) - get hasProof(): boolean - get isTxidOnly(): boolean - get txid(): string - get tx(): Transaction | undefined - get rawTx(): number[] | undefined - get rawTxUint8Array(): Uint8Array | undefined - constructor(tx: Transaction | Uint8Array | number[] | string, bumpIndex?: number) - static fromTx(tx: Transaction, bumpIndex?: number): BeefTx - static fromRawTx(rawTx: Uint8Array | number[], bumpIndex?: number): BeefTx - static fromTxid(txid: string, bumpIndex?: number): BeefTx - toWriter(writer: Writer | WriterUint8Array, version: number): void - static fromReader(br: Reader | ReaderUint8Array, version: number): BeefTx + _bumpIndex?: number + _tx?: Transaction + _rawTx?: Uint8Array + _txid?: string + inputTxids: string[] = [] + isValid?: boolean = undefined + get bumpIndex(): number | undefined + set bumpIndex(v: number | undefined) + get hasProof(): boolean + get isTxidOnly(): boolean + get txid(): string + get tx(): Transaction | undefined + get rawTx(): number[] | undefined + get rawTxUint8Array(): Uint8Array | undefined + syncRawTxFromTransaction(): boolean + constructor( + tx: Transaction | Uint8Array | number[] | string, + bumpIndex?: number, + inputTxids?: string[] + ) + static fromTx(tx: Transaction, bumpIndex?: number): BeefTx + static fromRawTx(rawTx: Uint8Array | number[], bumpIndex?: number): BeefTx + static fromTxid(txid: string, bumpIndex?: number): BeefTx + toWriter(writer: Writer | WriterUint8Array, version: number): void + static fromReader(br: Reader | ReaderUint8Array, version: number): BeefTx } ``` @@ -1224,16 +1595,17 @@ See also: [Reader](./primitives.md#class-reader), [ReaderUint8Array](./primitive #### Constructor ```ts -constructor(tx: Transaction | Uint8Array | number[] | string, bumpIndex?: number) +constructor(tx: Transaction | Uint8Array | number[] | string, bumpIndex?: number, inputTxids?: string[]) ``` + See also: [Transaction](./transaction.md#class-transaction) Argument Details -+ **tx** - + If string, must be a valid txid. If `number[]` must be a valid serialized transaction. -+ **bumpIndex** - + If transaction already has a proof in the beef to which it will be added. +- **tx** + - If string, must be a valid txid. If `number[]` must be a valid serialized transaction. +- **bumpIndex** + - If transaction already has a proof in the beef to which it will be added. #### Property isValid @@ -1245,17 +1617,47 @@ Typically set by sorting transactions by proven dependency chains. isValid?: boolean = undefined ``` +#### Method syncRawTxFromTransaction + +Synchronizes a nested transaction after mutation through the normal +Transaction APIs. Returns true when its serialized identity or dependencies +changed. + +```ts +syncRawTxFromTransaction(): boolean +``` + Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Class: EvidenceScriptWork + +Internal cache. Only coordinator-owned transactions are bound to its scope below. + +```ts +export class EvidenceScriptWork { + constructor(private readonly limits: Readonly) + clear(): void + inputs(scope: EvidenceScriptScope, params: BdkVerifyScriptsParams, verify: (skipScripts: boolean) => InputResult): InputResult + async batch(scope: EvidenceScriptScope, params: readonly BdkVerifyScriptsParams[], backend: BdkVerifierInterface): Promise +} +``` + +See also: [BdkVerifierInterface](./transaction.md#interface-bdkverifierinterface), [BdkVerifyScriptsParams](./transaction.md#interface-bdkverifyscriptsparams), [EvidenceScriptScope](./transaction.md#interface-evidencescriptscope), [TransactionEvidenceLimits](./transaction.md#interface-transactionevidencelimits), [verify](./compat.md#variable-verify) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Class: FetchHttpClient Adapter for Node Https module to be used as HttpClient ```ts export class FetchHttpClient implements HttpClient { - constructor(private readonly fetch: Fetch) - async request(url: string, options: HttpClientRequestOptions): Promise> + constructor(private readonly fetch: Fetch) + async request(url: string, options: HttpClientRequestOptions): Promise> } ``` @@ -1264,6 +1666,7 @@ See also: [Fetch](./transaction.md#type-fetch), [HttpClient](./transaction.md#in Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: LivePolicy Represents a live fee policy that fetches current rates from ARC GorillaPool. @@ -1271,9 +1674,9 @@ Extends SatoshisPerKilobyte to reuse transaction size calculation logic. ```ts export default class LivePolicy extends SatoshisPerKilobyte { - constructor(cacheValidityMs: number = 5 * 60 * 1000) - static getInstance(cacheValidityMs: number = 5 * 60 * 1000): LivePolicy - async computeFee(tx: Transaction): Promise + constructor(cacheValidityMs: number = 5 * 60 * 1000) + static getInstance(cacheValidityMs: number = 5 * 60 * 1000): LivePolicy + override async computeFee(tx: Transaction): Promise } ``` @@ -1284,13 +1687,13 @@ See also: [SatoshisPerKilobyte](./transaction.md#class-satoshisperkilobyte), [Tr Constructs an instance of the live policy fee model. ```ts -constructor(cacheValidityMs: number = 5 * 60 * 1000) +constructor(cacheValidityMs: number = 5 * 60 * 1000) ``` Argument Details -+ **cacheValidityMs** - + How long to cache the fee rate in milliseconds (default: 5 minutes) +- **cacheValidityMs** + - How long to cache the fee rate in milliseconds (default: 5 minutes) #### Method computeFee @@ -1298,8 +1701,9 @@ Computes the fee for a given transaction using the current live rate. Overrides the parent method to use dynamic rate fetching. ```ts -async computeFee(tx: Transaction): Promise +override async computeFee(tx: Transaction): Promise ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -1308,16 +1712,17 @@ The fee in satoshis for the transaction. Argument Details -+ **tx** - + The transaction for which a fee is to be computed. +- **tx** + - The transaction for which a fee is to be computed. #### Method getInstance Gets the singleton instance of LivePolicy to ensure cache sharing across the application. ```ts -static getInstance(cacheValidityMs: number = 5 * 60 * 1000): LivePolicy +static getInstance(cacheValidityMs: number = 5 * 60 * 1000): LivePolicy ``` + See also: [LivePolicy](./transaction.md#class-livepolicy) Returns @@ -1326,12 +1731,13 @@ The singleton LivePolicy instance Argument Details -+ **cacheValidityMs** - + How long to cache the fee rate in milliseconds (default: 5 minutes) +- **cacheValidityMs** + - How long to cache the fee rate in milliseconds (default: 5 minutes) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: MerklePath Represents a Merkle Path, which is used to provide a compact proof of inclusion for a @@ -1342,39 +1748,56 @@ Example ```ts // Creating and verifying a Merkle Path -const merklePath = MerklePath.fromHex('...'); -const isValid = merklePath.verify(txid, chainTracker); +const merklePath = MerklePath.fromHex('...') +const isValid = merklePath.verify(txid, chainTracker) ``` ```ts export default class MerklePath { - blockHeight: number; - path: Array>; - static fromHex(hex: string): MerklePath - static fromReader(reader: Reader | ReaderUint8Array, legalOffsetsOnly: boolean = true): MerklePath - static fromBinary(bump: number[] | Uint8Array): MerklePath - static fromCoinbaseTxidAndHeight(txid: string, height: number): MerklePath - constructor(blockHeight: number, path: Array>, legalOffsetsOnly: boolean = true) - toWriter(writer: Writer | WriterUint8Array): void - toBinary(): number[] - toBinaryUint8Array(): Uint8Array - toHex(): string - computeRoot(txid?: string): string - findOrComputeLeaf(height: number, offset: number): MerklePathLeaf | undefined - async verify(txid: string, chainTracker: ChainTracker): Promise - combine(other: MerklePath): void - trim(): void - extract(txids: string[]): MerklePath + blockHeight: number + path: Array< + Array<{ + offset: number + hash?: string + txid?: boolean + duplicate?: boolean + }> + > + static fromHex(hex: string): MerklePath + static fromReader( + reader: Reader | ReaderUint8Array, + legalOffsetsOnly: boolean = true, + validateRoots: boolean = true + ): MerklePath + static fromBinary( + bump: number[] | Uint8Array, + legalOffsetsOnly: boolean = true, + validateRoots: boolean = true + ): MerklePath + static fromCoinbaseTxidAndHeight(txid: string, height: number): MerklePath + constructor( + blockHeight: number, + path: Array< + Array<{ + offset: number + hash?: string + txid?: boolean + duplicate?: boolean + }> + >, + legalOffsetsOnly: boolean = true, + validateRoots: boolean = true + ) + toWriter(writer: Writer | WriterUint8Array): void + toBinary(): number[] + toBinaryUint8Array(): Uint8Array + toHex(): string + computeRoot(txid?: string): string + findOrComputeLeaf(height: number, offset: number): MerklePathLeaf | undefined + async verify(txid: string, chainTracker: ChainTracker): Promise + combine(other: MerklePath): void + trim(): void + extract(txids: string[]): MerklePath } ``` @@ -1385,14 +1808,15 @@ See also: [ChainTracker](./transaction.md#interface-chaintracker), [MerklePathLe Combines this MerklePath with another to create a compound proof. ```ts -combine(other: MerklePath): void +combine(other: MerklePath): void ``` + See also: [MerklePath](./transaction.md#class-merklepath) Argument Details -+ **other** - + Another MerklePath to combine with this path. +- **other** + - Another MerklePath to combine with this path. Throws @@ -1403,7 +1827,7 @@ Throws Computes the Merkle root from the provided transaction ID. ```ts -computeRoot(txid?: string): string +computeRoot(txid?: string): string ``` Returns @@ -1412,8 +1836,8 @@ Returns Argument Details -+ **txid** - + The transaction ID to compute the Merkle root for. If not provided, the root will be computed from an unspecified branch, and not all branches will be validated! +- **txid** + - The transaction ID to compute the Merkle root for. If not provided, the root will be computed from an unspecified branch, and not all branches will be validated! Throws @@ -1431,8 +1855,9 @@ into a single trimmed compound path. The extracted path is verified to compute the same Merkle root as the source. ```ts -extract(txids: string[]): MerklePath +extract(txids: string[]): MerklePath ``` + See also: [MerklePath](./transaction.md#class-merklepath) Returns @@ -1441,8 +1866,8 @@ Returns Argument Details -+ **txids** - + Transaction IDs to extract proofs for. +- **txids** + - Transaction IDs to extract proofs for. Throws @@ -1465,8 +1890,9 @@ Find leaf with `offset` at `height` or compute from level below, recursively. Does not add computed leaves to path. ```ts -findOrComputeLeaf(height: number, offset: number): MerklePathLeaf | undefined +findOrComputeLeaf(height: number, offset: number): MerklePathLeaf | undefined ``` + See also: [MerklePathLeaf](./transaction.md#interface-merklepathleaf) #### Method fromBinary @@ -1474,8 +1900,9 @@ See also: [MerklePathLeaf](./transaction.md#interface-merklepathleaf) Creates a MerklePath instance from a binary array. ```ts -static fromBinary(bump: number[] | Uint8Array): MerklePath +static fromBinary(bump: number[] | Uint8Array, legalOffsetsOnly: boolean = true, validateRoots: boolean = true): MerklePath ``` + See also: [MerklePath](./transaction.md#class-merklepath) Returns @@ -1484,14 +1911,15 @@ Returns Argument Details -+ **bump** - + The binary array representation of the Merkle Path. +- **bump** + - The binary array representation of the Merkle Path. #### Method fromCoinbaseTxidAndHeight ```ts -static fromCoinbaseTxidAndHeight(txid: string, height: number): MerklePath +static fromCoinbaseTxidAndHeight(txid: string, height: number): MerklePath ``` + See also: [MerklePath](./transaction.md#class-merklepath) Returns @@ -1500,18 +1928,19 @@ Returns Argument Details -+ **txid** - + The coinbase txid. -+ **height** - + The height of the block. +- **txid** + - The coinbase txid. +- **height** + - The height of the block. #### Method fromHex Creates a MerklePath instance from a hexadecimal string. ```ts -static fromHex(hex: string): MerklePath +static fromHex(hex: string): MerklePath ``` + See also: [MerklePath](./transaction.md#class-merklepath) Returns @@ -1520,15 +1949,15 @@ Returns Argument Details -+ **hex** - + The hexadecimal string representation of the Merkle Path. +- **hex** + - The hexadecimal string representation of the Merkle Path. #### Method toBinary Converts the MerklePath to a binary array format. ```ts -toBinary(): number[] +toBinary(): number[] ``` Returns @@ -1540,7 +1969,7 @@ Returns Converts the MerklePath to a binary array format. ```ts -toBinaryUint8Array(): Uint8Array +toBinaryUint8Array(): Uint8Array ``` Returns @@ -1552,7 +1981,7 @@ Returns Converts the MerklePath to a hexadecimal string format. ```ts -toHex(): string +toHex(): string ``` Returns @@ -1564,14 +1993,15 @@ Returns Serializes the MerklePath to the writer provided. ```ts -toWriter(writer: Writer | WriterUint8Array): void +toWriter(writer: Writer | WriterUint8Array): void ``` + See also: [Writer](./primitives.md#class-writer), [WriterUint8Array](./primitives.md#class-writeruint8array) Argument Details -+ **writer** - + The writer to which the Merkle Path will be serialized. +- **writer** + - The writer to which the Merkle Path will be serialized. #### Method trim @@ -1580,7 +2010,7 @@ Assumes that at least all required nodes are present. Leaves all levels sorted by increasing offset. ```ts -trim(): void +trim(): void ``` #### Method verify @@ -1588,8 +2018,9 @@ trim(): void Verifies if the given transaction ID is part of the Merkle tree at the specified block height. ```ts -async verify(txid: string, chainTracker: ChainTracker): Promise +async verify(txid: string, chainTracker: ChainTracker): Promise ``` + See also: [ChainTracker](./transaction.md#interface-chaintracker) Returns @@ -1598,22 +2029,23 @@ Returns Argument Details -+ **txid** - + The transaction ID to verify. -+ **chainTracker** - + The ChainTracker instance used to verify the Merkle root. +- **txid** + - The transaction ID to verify. +- **chainTracker** + - The ChainTracker instance used to verify the Merkle root. Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: NodejsHttpClient Adapter for Node Https module to be used as HttpClient ```ts export class NodejsHttpClient implements HttpClient { - constructor(private readonly https: HttpsNodejs) - async request(url: string, requestOptions: HttpClientRequestOptions): Promise + constructor(private readonly https: HttpsNodejs) + async request(url: string, requestOptions: HttpClientRequestOptions): Promise } ``` @@ -1622,15 +2054,16 @@ See also: [HttpClient](./transaction.md#interface-httpclient), [HttpClientReques Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: SatoshisPerKilobyte Represents the "satoshis per kilobyte" transaction fee model. ```ts export default class SatoshisPerKilobyte implements FeeModel { - value: number; - constructor(value: number) - async computeFee(tx: Transaction): Promise + value: number + constructor(value: number) + async computeFee(tx: Transaction): Promise } ``` @@ -1641,21 +2074,22 @@ See also: [FeeModel](./transaction.md#interface-feemodel), [Transaction](./trans Constructs an instance of the sat/kb fee model. ```ts -constructor(value: number) +constructor(value: number) ``` Argument Details -+ **value** - + The number of satoshis per kilobyte to charge as a fee. +- **value** + - The number of satoshis per kilobyte to charge as a fee. #### Method computeFee Computes the fee for a given transaction. ```ts -async computeFee(tx: Transaction): Promise +async computeFee(tx: Transaction): Promise ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -1664,12 +2098,13 @@ The fee in satoshis for the transaction, as a BigNumber. Argument Details -+ **tx** - + The transaction for which a fee is to be computed. +- **tx** + - The transaction for which a fee is to be computed. Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Class: Transaction Represents a complete Bitcoin transaction. This class encapsulates all the details @@ -1690,80 +2125,113 @@ await tx.broadcast(); ```ts export default class Transaction { - version: number; - inputs: TransactionInput[]; - outputs: TransactionOutput[]; - lockTime: number; - metadata: Record; - merklePath?: MerklePath; - static fromBEEF(beef: number[] | Uint8Array, txid?: string): Transaction - static fromAtomicBEEF(beef: number[] | Uint8Array): Transaction - static fromEF(ef: number[] | Uint8Array): Transaction - static parseScriptOffsets(bin: number[] | Uint8Array): { - inputs: Array<{ - vin: number; - offset: number; - length: number; - }>; - outputs: Array<{ - vout: number; - offset: number; - length: number; - }>; - } - static fromReader(br: Reader | ReaderUint8Array): Transaction - static fromBinary(bin: number[] | Uint8Array): Transaction - static fromHex(hex: string): Transaction - static fromHexEF(hex: string): Transaction - static fromHexBEEF(hex: string, txid?: string): Transaction - constructor(version: number = 1, inputs: TransactionInput[] = [], outputs: TransactionOutput[] = [], lockTime: number = 0, metadata: Record = new Map(), merklePath?: MerklePath) - addInput(input: TransactionInput): void - addOutput(output: TransactionOutput): void - addP2PKHOutput(address: number[] | string, satoshis?: number): void - updateMetadata(metadata: Record): void - async fee(modelOrFee: FeeModel | number = LivePolicy.getInstance(), changeDistribution: "equal" | "random" = "equal"): Promise - getFee(): number - async sign(): Promise - async broadcast(broadcaster: Broadcaster = defaultBroadcaster()): Promise - toBinary(): number[] - toUint8Array(): Uint8Array - toEF(): number[] - toEFUint8Array(): Uint8Array - toEFBinary(): Uint8Array - toHexEF(): string - toHex(): string - toHexBEEF(): string - toHexAtomicBEEF(): string - hash(enc?: "hex"): number[] | string - id(): number[]; - id(enc: "hex"): string; - id(enc?: "hex"): number[] | string - async verify(chainTracker: ChainTracker | "scripts only" = defaultChainTracker(), feeModel?: FeeModel, memoryLimit?: number): Promise - writeSerializedBEEF(writer: Writer | WriterUint8Array, allowPartial?: boolean): void - toBEEF(allowPartial?: boolean): number[] - toBEEFUint8Array(allowPartial?: boolean): Uint8Array - toAtomicBEEF(allowPartial?: boolean): number[] - toAtomicBEEFUint8Array(allowPartial?: boolean): Uint8Array - async completeWithWallet(wallet: WalletInterface, actionDescription?: DescriptionString5to50Bytes, originator?: string, options?: CreateActionOptions): Promise - preimage(inputIndex?: number, signatureScope?: number, subscript?: LockingScript): number[] + version: number + inputs: TransactionInput[] + outputs: TransactionOutput[] + lockTime: number + metadata: Record + merklePath?: MerklePath + getSignatureHashCache(): SignatureHashCache + materializeSourceTXIDs(): void + static fromBEEF(beef: number[] | Uint8Array, txid?: string): Transaction + static fromBEEFView(beef: Uint8Array, txid?: string): Transaction + static fromAtomicBEEF(beef: number[] | Uint8Array): Transaction + static fromAtomicBEEFView(beef: Uint8Array): Transaction + static fromEF(ef: number[] | Uint8Array): Transaction + static parseScriptOffsets(bin: number[] | Uint8Array): { + inputs: Array<{ + vin: number + offset: number + length: number + }> + outputs: Array<{ + vout: number + offset: number + length: number + }> + } + static fromReader(br: Reader | ReaderUint8Array): Transaction + static fromBinary(bin: number[] | Uint8Array): Transaction + static fromBinaryView(bin: Uint8Array): Transaction + static fromHex(hex: string): Transaction + static fromHexEF(hex: string): Transaction + static fromHexBEEF(hex: string, txid?: string): Transaction + constructor( + version: number = 1, + inputs: TransactionInput[] = [], + outputs: TransactionOutput[] = [], + lockTime: number = 0, + metadata: Record = new Map(), + merklePath?: MerklePath + ) + addInput(input: TransactionInput): void + addOutput(output: TransactionOutput): void + addP2PKHOutput(address: number[] | string, satoshis?: number): void + updateMetadata(metadata: Record): void + async fee( + modelOrFee: FeeModel | number = LivePolicy.getInstance(), + changeDistribution: 'equal' | 'random' = 'equal' + ): Promise + getFee(): number + async sign( + options: { + skipExistingSignatures?: boolean + } = {} + ): Promise + async broadcast( + broadcaster: Broadcaster = defaultBroadcaster() + ): Promise + toBinary(): number[] + toUint8Array(): Uint8Array + toEF(): number[] + toEFUint8Array(): Uint8Array + toEFBinary(): Uint8Array + toHexEF(): string + toHex(): string + toHexBEEF(): string + toHexAtomicBEEF(): string + hash(enc?: 'hex'): number[] | string + id(): number[] + id(enc: 'hex'): string + id(enc?: 'hex'): number[] | string + async verify( + chainTracker: ChainTracker | 'scripts only' = defaultChainTracker(), + feeModel?: FeeModel, + memoryLimit?: number, + verifier?: BdkVerifierInterface + ): Promise + writeSerializedBEEF(writer: Writer | WriterUint8Array, allowPartial?: boolean): void + toBEEF(allowPartial?: boolean): number[] + toBEEFUint8Array(allowPartial?: boolean): Uint8Array + toBEEFBytes(allowPartial?: boolean): Uint8Array + toAtomicBEEF(allowPartial?: boolean): number[] + toAtomicBEEFUint8Array(allowPartial?: boolean): Uint8Array + async completeWithWallet( + wallet: WalletInterface, + actionDescription?: DescriptionString5to50Bytes, + originator?: string, + options?: CreateActionOptions + ): Promise + preimage(inputIndex?: number, signatureScope?: number, subscript?: LockingScript): number[] } ``` -See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [BroadcastResponse](./transaction.md#interface-broadcastresponse), [Broadcaster](./transaction.md#interface-broadcaster), [ChainTracker](./transaction.md#interface-chaintracker), [CreateActionOptions](./wallet.md#interface-createactionoptions), [DescriptionString5to50Bytes](./wallet.md#type-descriptionstring5to50bytes), [FeeModel](./transaction.md#interface-feemodel), [LivePolicy](./transaction.md#class-livepolicy), [LockingScript](./script.md#class-lockingscript), [MerklePath](./transaction.md#class-merklepath), [Reader](./primitives.md#class-reader), [ReaderUint8Array](./primitives.md#class-readeruint8array), [TransactionInput](./transaction.md#interface-transactioninput), [TransactionOutput](./transaction.md#interface-transactionoutput), [WalletInterface](./wallet.md#interface-walletinterface), [Writer](./primitives.md#class-writer), [WriterUint8Array](./primitives.md#class-writeruint8array), [defaultBroadcaster](./transaction.md#function-defaultbroadcaster), [defaultChainTracker](./transaction.md#function-defaultchaintracker), [sign](./compat.md#variable-sign), [toHex](./primitives.md#variable-tohex), [toUint8Array](./primitives.md#variable-touint8array), [verify](./compat.md#variable-verify) +See also: [BdkVerifierInterface](./transaction.md#interface-bdkverifierinterface), [BroadcastFailure](./transaction.md#interface-broadcastfailure), [BroadcastResponse](./transaction.md#interface-broadcastresponse), [Broadcaster](./transaction.md#interface-broadcaster), [ChainTracker](./transaction.md#interface-chaintracker), [CreateActionOptions](./wallet.md#interface-createactionoptions), [DescriptionString5to50Bytes](./wallet.md#type-descriptionstring5to50bytes), [FeeModel](./transaction.md#interface-feemodel), [LivePolicy](./transaction.md#class-livepolicy), [LockingScript](./script.md#class-lockingscript), [MerklePath](./transaction.md#class-merklepath), [Reader](./primitives.md#class-reader), [ReaderUint8Array](./primitives.md#class-readeruint8array), [SignatureHashCache](./primitives.md#interface-signaturehashcache), [TransactionInput](./transaction.md#interface-transactioninput), [TransactionOutput](./transaction.md#interface-transactionoutput), [WalletInterface](./wallet.md#interface-walletinterface), [Writer](./primitives.md#class-writer), [WriterUint8Array](./primitives.md#class-writeruint8array), [defaultBroadcaster](./transaction.md#function-defaultbroadcaster), [defaultChainTracker](./transaction.md#function-defaultchaintracker), [sign](./compat.md#variable-sign), [toHex](./primitives.md#variable-tohex), [toUint8Array](./primitives.md#variable-touint8array), [verify](./compat.md#variable-verify) #### Method addInput Adds a new input to the transaction. ```ts -addInput(input: TransactionInput): void +addInput(input: TransactionInput): void ``` + See also: [TransactionInput](./transaction.md#interface-transactioninput) Argument Details -+ **input** - + The TransactionInput object to add to the transaction. +- **input** + - The TransactionInput object to add to the transaction. Throws @@ -1774,37 +2242,39 @@ Throws Adds a new output to the transaction. ```ts -addOutput(output: TransactionOutput): void +addOutput(output: TransactionOutput): void ``` + See also: [TransactionOutput](./transaction.md#interface-transactionoutput) Argument Details -+ **output** - + The TransactionOutput object to add to the transaction. +- **output** + - The TransactionOutput object to add to the transaction. #### Method addP2PKHOutput Adds a new P2PKH output to the transaction. ```ts -addP2PKHOutput(address: number[] | string, satoshis?: number): void +addP2PKHOutput(address: number[] | string, satoshis?: number): void ``` Argument Details -+ **address** - + The P2PKH address of the output. -+ **satoshis** - + The number of satoshis to send to the address - if not provided, the output is considered a change output. +- **address** + - The P2PKH address of the output. +- **satoshis** + - The number of satoshis to send to the address - if not provided, the output is considered a change output. #### Method broadcast Broadcasts a transaction. ```ts -async broadcast(broadcaster: Broadcaster = defaultBroadcaster()): Promise +async broadcast(broadcaster: Broadcaster = defaultBroadcaster()): Promise ``` + See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [BroadcastResponse](./transaction.md#interface-broadcastresponse), [Broadcaster](./transaction.md#interface-broadcaster), [defaultBroadcaster](./transaction.md#function-defaultbroadcaster) Returns @@ -1813,8 +2283,8 @@ A BroadcastResponse or BroadcastFailure from the Broadcaster Argument Details -+ **broadcaster** - + The Broadcaster instance wwhere the transaction will be sent +- **broadcaster** + - The Broadcaster instance wwhere the transaction will be sent #### Method completeWithWallet @@ -1824,20 +2294,21 @@ transaction into a format that can be processed by the wallet, and then updates this transaction object with the result from the wallet. ```ts -async completeWithWallet(wallet: WalletInterface, actionDescription?: DescriptionString5to50Bytes, originator?: string, options?: CreateActionOptions): Promise +async completeWithWallet(wallet: WalletInterface, actionDescription?: DescriptionString5to50Bytes, originator?: string, options?: CreateActionOptions): Promise ``` + See also: [CreateActionOptions](./wallet.md#interface-createactionoptions), [DescriptionString5to50Bytes](./wallet.md#type-descriptionstring5to50bytes), [WalletInterface](./wallet.md#interface-walletinterface) Argument Details -+ **wallet** - + The BRC-100 compliant wallet to use for completing the transaction -+ **actionDescription** - + Optional description for the action -+ **originator** - + Optional originator domain name -+ **options** - + Optional settings for transaction creation (e.g., acceptDelayedBroadcast, trustSelf, noSend, etc.) +- **wallet** + - The BRC-100 compliant wallet to use for completing the transaction +- **actionDescription** + - Optional description for the action +- **originator** + - Optional originator domain name +- **options** + - Optional settings for transaction creation (e.g., acceptDelayedBroadcast, trustSelf, noSend, etc.) #### Method fee @@ -1846,17 +2317,18 @@ If no fee model is provided, uses a LivePolicy fee model that fetches current ra If fee is a number, the transaction uses that value as fee. ```ts -async fee(modelOrFee: FeeModel | number = LivePolicy.getInstance(), changeDistribution: "equal" | "random" = "equal"): Promise +async fee(modelOrFee: FeeModel | number = LivePolicy.getInstance(), changeDistribution: "equal" | "random" = "equal"): Promise ``` + See also: [FeeModel](./transaction.md#interface-feemodel), [LivePolicy](./transaction.md#class-livepolicy) Argument Details -+ **modelOrFee** - + The initialized fee model to use or fixed fee for the transaction -+ **changeDistribution** - + Specifies how the change should be distributed -amongst the change outputs +- **modelOrFee** + - The initialized fee model to use or fixed fee for the transaction +- **changeDistribution** + - Specifies how the change should be distributed + amongst the change outputs #### Method fromAtomicBEEF @@ -1864,8 +2336,9 @@ Creates a new transaction from an Atomic BEEF (BRC-95) structure. Extracts the subject transaction and supporting merkle path and source transactions contained in the BEEF data ```ts -static fromAtomicBEEF(beef: number[] | Uint8Array): Transaction +static fromAtomicBEEF(beef: number[] | Uint8Array): Transaction ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -1874,8 +2347,8 @@ The subject transaction, linked to its associated inputs populated with merkle p Argument Details -+ **beef** - + A binary representation of an Atomic BEEF structure. +- **beef** + - A binary representation of an Atomic BEEF structure. #### Method fromBEEF @@ -1885,8 +2358,9 @@ If the TXID is provided but not found in the BEEF data, an error will be thrown. If no TXID is provided, the last transaction in the BEEF data is returned, or the atomic txid. ```ts -static fromBEEF(beef: number[] | Uint8Array, txid?: string): Transaction +static fromBEEF(beef: number[] | Uint8Array, txid?: string): Transaction ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -1895,18 +2369,19 @@ An anchored transaction, linked to its associated inputs populated with merkle p Argument Details -+ **beef** - + A binary representation of transactions in BEEF format. -+ **txid** - + Optional TXID of the transaction to retrieve from the BEEF data. +- **beef** + - A binary representation of transactions in BEEF format. +- **txid** + - Optional TXID of the transaction to retrieve from the BEEF data. #### Method fromBinary Creates a Transaction instance from a binary array. ```ts -static fromBinary(bin: number[] | Uint8Array): Transaction +static fromBinary(bin: number[] | Uint8Array): Transaction ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -1915,16 +2390,28 @@ Returns Argument Details -+ **bin** - + The binary array representation of the transaction. +- **bin** + - The binary array representation of the transaction. + +#### Method fromBinaryView + +Parses a transaction while retaining zero-copy views over `bin` for the raw +transaction and its scripts. The caller must not mutate `bin`. + +```ts +static fromBinaryView(bin: Uint8Array): Transaction +``` + +See also: [Transaction](./transaction.md#class-transaction) #### Method fromEF Creates a new transaction, linked to its inputs and their associated merkle paths, from a EF (BRC-30) structure. ```ts -static fromEF(ef: number[] | Uint8Array): Transaction +static fromEF(ef: number[] | Uint8Array): Transaction ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -1933,16 +2420,17 @@ An extended transaction, linked to its associated inputs by locking script and s Argument Details -+ **ef** - + A binary representation of a transaction in EF format. +- **ef** + - A binary representation of a transaction in EF format. #### Method fromHex Creates a Transaction instance from a hexadecimal string. ```ts -static fromHex(hex: string): Transaction +static fromHex(hex: string): Transaction ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -1951,8 +2439,8 @@ Returns Argument Details -+ **hex** - + The hexadecimal string representation of the transaction. +- **hex** + - The hexadecimal string representation of the transaction. #### Method fromHexBEEF @@ -1962,8 +2450,9 @@ If the TXID is provided but not found in the BEEF data, an error will be thrown. If no TXID is provided, the last transaction in the BEEF data is returned. ```ts -static fromHexBEEF(hex: string, txid?: string): Transaction +static fromHexBEEF(hex: string, txid?: string): Transaction ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -1972,18 +2461,19 @@ Returns Argument Details -+ **hex** - + The hexadecimal string representation of the transaction BEEF. -+ **txid** - + Optional TXID of the transaction to retrieve from the BEEF data. +- **hex** + - The hexadecimal string representation of the transaction BEEF. +- **txid** + - Optional TXID of the transaction to retrieve from the BEEF data. #### Method fromHexEF Creates a Transaction instance from a hexadecimal string encoded EF. ```ts -static fromHexEF(hex: string): Transaction +static fromHexEF(hex: string): Transaction ``` + See also: [Transaction](./transaction.md#class-transaction) Returns @@ -1992,27 +2482,38 @@ Returns Argument Details -+ **hex** - + The hexadecimal string representation of the transaction EF. +- **hex** + - The hexadecimal string representation of the transaction EF. #### Method getFee Utility method that returns the current fee based on inputs and outputs ```ts -getFee(): number +getFee(): number ``` Returns The current transaction fee +#### Method getSignatureHashCache + +Returns the transaction-wide signature hash cache active during signing. +Callers outside a signing operation receive an isolated cache. + +```ts +getSignatureHashCache(): SignatureHashCache +``` + +See also: [SignatureHashCache](./primitives.md#interface-signaturehashcache) + #### Method hash Calculates the transaction's hash. ```ts -hash(enc?: "hex"): number[] | string +hash(enc?: "hex"): number[] | string ``` Returns @@ -2021,8 +2522,8 @@ Returns Argument Details -+ **enc** - + The encoding to use for the hash. If 'hex', returns a hexadecimal string; otherwise returns a binary array. +- **enc** + - The encoding to use for the hash. If 'hex', returns a hexadecimal string; otherwise returns a binary array. #### Method id @@ -2050,15 +2551,15 @@ Returns Argument Details -+ **enc** - + The encoding to use for the ID. If 'hex', returns a hexadecimal string. +- **enc** + - The encoding to use for the ID. If 'hex', returns a hexadecimal string. #### Method id Calculates the transaction's ID. ```ts -id(enc?: "hex"): number[] | string +id(enc?: "hex"): number[] | string ``` Returns @@ -2067,8 +2568,17 @@ Returns Argument Details -+ **enc** - + The encoding to use for the ID. If 'hex', returns a hexadecimal string; otherwise returns a binary array. +- **enc** + - The encoding to use for the ID. If 'hex', returns a hexadecimal string; otherwise returns a binary array. + +#### Method materializeSourceTXIDs + +Iteratively materializes source transaction IDs so deep spend chains do not +recurse through `hash()` while serializing their parents. + +```ts +materializeSourceTXIDs(): void +``` #### Method parseScriptOffsets @@ -2092,7 +2602,7 @@ static parseScriptOffsets(bin: number[] | Uint8Array): { offset: number; length: number; }>; -} +} ``` Returns @@ -2103,16 +2613,17 @@ outputs: { vout: number, offset: number, length: number }[] Argument Details -+ **bin** - + binary transaction data +- **bin** + - binary transaction data #### Method preimage Returns the formatted preimage of a transaction for the requested input index, signature scope (default SIGHASH_FORKID | SIGHASH_ALL), and optional subscript. ```ts -preimage(inputIndex?: number, signatureScope?: number, subscript?: LockingScript): number[] +preimage(inputIndex?: number, signatureScope?: number, subscript?: LockingScript): number[] ``` + See also: [LockingScript](./script.md#class-lockingscript) Returns @@ -2121,21 +2632,28 @@ The formatted preimage Argument Details -+ **inputIndex** - + The index of the input to generate the preimage for -+ **signatureScope** - + The signature scope to use for the preimage -+ **subscript** - + The subscript to use for the preimage (optional) +- **inputIndex** + - The index of the input to generate the preimage for +- **signatureScope** + - The signature scope to use for the preimage +- **subscript** + - The subscript to use for the preimage (optional) #### Method sign Signs a transaction, hydrating all its unlocking scripts based on the provided script templates where they are available. ```ts -async sign(): Promise +async sign(options: { + skipExistingSignatures?: boolean; +} = {}): Promise ``` +Argument Details + +- **options** + - Signing behavior. Set `skipExistingSignatures` to preserve inputs that already have an unlocking script. + #### Method toAtomicBEEF Serializes this transaction and its inputs into the Atomic BEEF (BRC-95) format. @@ -2144,7 +2662,7 @@ and then the BEEF data containing only the subject transaction and its dependenc This format ensures that the BEEF structure is atomic and contains no unrelated transactions. ```ts -toAtomicBEEF(allowPartial?: boolean): number[] +toAtomicBEEF(allowPartial?: boolean): number[] ``` Returns @@ -2153,8 +2671,8 @@ Returns Argument Details -+ **allowPartial** - + If true, error will not be thrown if there are any missing sourceTransactions. +- **allowPartial** + - If true, error will not be thrown if there are any missing sourceTransactions. Throws @@ -2168,7 +2686,7 @@ and then the BEEF data containing only the subject transaction and its dependenc This format ensures that the BEEF structure is atomic and contains no unrelated transactions. ```ts -toAtomicBEEFUint8Array(allowPartial?: boolean): Uint8Array +toAtomicBEEFUint8Array(allowPartial?: boolean): Uint8Array ``` Returns @@ -2177,8 +2695,8 @@ Returns Argument Details -+ **allowPartial** - + If true, error will not be thrown if there are any missing sourceTransactions. +- **allowPartial** + - If true, error will not be thrown if there are any missing sourceTransactions. Throws @@ -2189,7 +2707,7 @@ Error if there are any missing sourceTransactions unless `allowPartial` is true. Serializes this transaction, together with its inputs and the respective merkle proofs, into the BEEF (BRC-62) format. This enables efficient verification of its compliance with the rules of SPV. ```ts -toBEEF(allowPartial?: boolean): number[] +toBEEF(allowPartial?: boolean): number[] ``` Returns @@ -2198,19 +2716,27 @@ The serialized BEEF structure Argument Details -+ **allowPartial** - + If true, error will not be thrown if there are any missing sourceTransactions. +- **allowPartial** + - If true, error will not be thrown if there are any missing sourceTransactions. Throws Error if there are any missing sourceTransactions unless `allowPartial` is true. +#### Method toBEEFBytes + +Serializes BEEF to a real typed byte array. + +```ts +toBEEFBytes(allowPartial?: boolean): Uint8Array +``` + #### Method toBEEFUint8Array Serializes this transaction, together with its inputs and the respective merkle proofs, into the BEEF (BRC-62) format. This enables efficient verification of its compliance with the rules of SPV. ```ts -toBEEFUint8Array(allowPartial?: boolean): Uint8Array +toBEEFUint8Array(allowPartial?: boolean): Uint8Array ``` Returns @@ -2219,8 +2745,8 @@ The serialized BEEF structure Argument Details -+ **allowPartial** - + If true, error will not be thrown if there are any missing sourceTransactions. +- **allowPartial** + - If true, error will not be thrown if there are any missing sourceTransactions. Throws @@ -2231,7 +2757,7 @@ Error if there are any missing sourceTransactions unless `allowPartial` is true. Converts the transaction to a binary array format. ```ts -toBinary(): number[] +toBinary(): number[] ``` Returns @@ -2243,45 +2769,43 @@ Returns Converts the transaction to a BRC-30 EF format. ```ts -toEF(): number[] +toEF(): number[] ``` Returns - The BRC-30 EF representation of the transaction. -#### Method toEFUint8Array +#### Method toEFBinary -Converts the transaction to a BRC-30 EF format. +Converts the transaction to a memoized BRC-30 EF byte array. ```ts -toEFUint8Array(): Uint8Array +toEFBinary(): Uint8Array ``` Returns -- The BRC-30 EF representation of the transaction. +The cached BRC-30 EF representation. -#### Method toEFBinary +#### Method toEFUint8Array -Converts the transaction to a memoized BRC-30 EF byte array. The returned view -is reused until transaction or referenced source-output serialization state -changes and must be treated as immutable; call `.slice()` for a mutable copy. +Converts the transaction to a BRC-30 EF format. ```ts -toEFBinary(): Uint8Array +toEFUint8Array(): Uint8Array ``` Returns -The cached BRC-30 EF representation. +- The BRC-30 EF representation of the transaction. #### Method toHex Converts the transaction to a hexadecimal string format. ```ts -toHex(): string +toHex(): string ``` Returns @@ -2293,7 +2817,7 @@ Returns Converts the transaction to a hexadecimal string Atomic BEEF. ```ts -toHexAtomicBEEF(): string +toHexAtomicBEEF(): string ``` Returns @@ -2305,7 +2829,7 @@ Returns Converts the transaction to a hexadecimal string BEEF. ```ts -toHexBEEF(): string +toHexBEEF(): string ``` Returns @@ -2317,7 +2841,7 @@ Returns Converts the transaction to a hexadecimal string EF. ```ts -toHexEF(): string +toHexEF(): string ``` Returns @@ -2329,22 +2853,23 @@ Returns Updates the transaction's metadata. ```ts -updateMetadata(metadata: Record): void +updateMetadata(metadata: Record): void ``` Argument Details -+ **metadata** - + The metadata object to merge into the existing metadata. +- **metadata** + - The metadata object to merge into the existing metadata. #### Method verify Verifies the legitimacy of the Bitcoin transaction according to the rules of SPV by ensuring all the input transactions link back to valid block headers, the chain of spends for all inputs are valid, and the sum of inputs is not less than the sum of outputs. ```ts -async verify(chainTracker: ChainTracker | "scripts only" = defaultChainTracker(), feeModel?: FeeModel, memoryLimit?: number): Promise +async verify(chainTracker: ChainTracker | "scripts only" = defaultChainTracker(), feeModel?: FeeModel, memoryLimit?: number, verifier?: BdkVerifierInterface): Promise ``` -See also: [ChainTracker](./transaction.md#interface-chaintracker), [FeeModel](./transaction.md#interface-feemodel), [defaultChainTracker](./transaction.md#function-defaultchaintracker) + +See also: [BdkVerifierInterface](./transaction.md#interface-bdkverifierinterface), [ChainTracker](./transaction.md#interface-chaintracker), [FeeModel](./transaction.md#interface-feemodel), [defaultChainTracker](./transaction.md#function-defaultchaintracker) Returns @@ -2352,13 +2877,16 @@ Whether the transaction is valid according to the rules of SPV. Argument Details -+ **chainTracker** - + An instance of ChainTracker, a Bitcoin block header tracker. If the value is set to 'scripts only', headers will not be verified. If not provided then the default chain tracker will be used. -+ **feeModel** - + An instance of FeeModel, a fee model to use for fee calculation. If not provided then the default fee model will be used. -+ **memoryLimit** - + Optional caller-supplied local interpreter memory budget. If omitted, -post-Genesis validation does not impose an arbitrary SDK memory cap. +- **chainTracker** + - An instance of ChainTracker, a Bitcoin block header tracker. If the value is set to 'scripts only', headers will not be verified. If not provided then the default chain tracker will be used. +- **feeModel** + - An instance of FeeModel, a fee model to use for fee calculation. If not provided then the default fee model will be used. +- **memoryLimit** + - Optional caller-supplied local script-interpreter + memory budget. If omitted, post-Genesis validation does not impose an + arbitrary SDK memory cap. +- **verifier** + - An optional asynchronous script backend. Adaptive backends may decline before execution to preserve the JavaScript path. Example @@ -2371,8 +2899,9 @@ tx.verify(new WhatsOnChain(), LivePolicy.getInstance()) Serializes this transaction, together with its inputs and the respective merkle proofs, into the BEEF (BRC-62) format. This enables efficient verification of its compliance with the rules of SPV. ```ts -writeSerializedBEEF(writer: Writer | WriterUint8Array, allowPartial?: boolean): void +writeSerializedBEEF(writer: Writer | WriterUint8Array, allowPartial?: boolean): void ``` + See also: [Writer](./primitives.md#class-writer), [WriterUint8Array](./primitives.md#class-writeruint8array) Returns @@ -2381,10 +2910,10 @@ The serialized BEEF structure Argument Details -+ **writer** - + The writer to serialize to -+ **allowPartial** - + If true, error will not be thrown if there are any missing sourceTransactions. +- **writer** + - The writer to serialize to +- **allowPartial** + - If true, error will not be thrown if there are any missing sourceTransactions. Throws @@ -2393,20 +2922,110 @@ Error if there are any missing sourceTransactions unless `allowPartial` is true. Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Class: TransactionEvidenceCoordinator + +Bounded, process-local transaction evidence work sharing. This is independent of +lookup services, certificates and trust ratings. Positive reuse always checks +canonical anchors again; ChainTracker remains the caller's trusted chain source. +Synchronous parsing/script execution is byte/memory bounded, not preemptible. + +```ts +export class TransactionEvidenceCoordinator { + readonly limits: Readonly + constructor(options: TransactionEvidenceCoordinatorOptions) + setContext(context: TransactionEvidenceContext): void + dispose(): void + async verify( + evidence: TransactionEvidence, + options: { + signal?: AbortSignal + } = {} + ): Promise + getStats(): { + pendingTransactions: number + consumers: number + cachedTransactions: number + retainedBytes: number + pendingChainCalls: number + activeAttempts: number + } +} +``` + +See also: [TransactionEvidence](./transaction.md#interface-transactionevidence), [TransactionEvidenceContext](./transaction.md#interface-transactionevidencecontext), [TransactionEvidenceCoordinatorOptions](./transaction.md#interface-transactionevidencecoordinatoroptions), [TransactionEvidenceLimits](./transaction.md#interface-transactionevidencelimits), [VerifiedTransactionOutput](./transaction.md#interface-verifiedtransactionoutput), [verify](./compat.md#variable-verify) + +#### Method getStats + +Payload-free local diagnostics; pending calls include abandoned, non-abortable I/O. + +```ts +getStats(): { + pendingTransactions: number; + consumers: number; + cachedTransactions: number; + retainedBytes: number; + pendingChainCalls: number; + activeAttempts: number; +} +``` + +#### Method setContext + +Explicit session/network/policy change. Stale in-flight work cannot publish. + +```ts +setContext(context: TransactionEvidenceContext): void +``` + +See also: [TransactionEvidenceContext](./transaction.md#interface-transactionevidencecontext) + +#### Method verify + +Snapshot intake is synchronous up to the returned Promise's first await. + +```ts +async verify(evidence: TransactionEvidence, options: { + signal?: AbortSignal; +} = {}): Promise +``` + +See also: [TransactionEvidence](./transaction.md#interface-transactionevidence), [VerifiedTransactionOutput](./transaction.md#interface-verifiedtransactionoutput) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Class: TransactionEvidenceError + +Bounded, payload-free outcome; rejected receipts never permanently reject a txid. + +```ts +export class TransactionEvidenceError extends Error { + constructor(public readonly code: TransactionEvidenceErrorCode) +} +``` + +See also: [TransactionEvidenceErrorCode](./transaction.md#type-transactionevidenceerrorcode) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Class: WhatsOnChain Represents a chain tracker based on What's On Chain . ```ts export default class WhatsOnChain implements ChainTracker { - readonly network: string; - readonly apiKey: string; - protected readonly URL: string; - protected readonly httpClient: HttpClient; - constructor(network: "main" | "test" | "stn" = "main", config: WhatsOnChainConfig = {}) - async isValidRootForHeight(root: string, height: number): Promise - async currentHeight(): Promise - protected getHttpHeaders(): Record + readonly network: string + readonly apiKey: string + protected readonly URL: string + protected readonly httpClient: HttpClient + constructor(network: 'main' | 'test' | 'stn' = 'main', config: WhatsOnChainConfig = {}) + async isValidRootForHeight(root: string, height: number): Promise + async currentHeight(): Promise + protected getHttpHeaders(): Record } ``` @@ -2417,30 +3036,48 @@ See also: [ChainTracker](./transaction.md#interface-chaintracker), [HttpClient]( Constructs an instance of the WhatsOnChain ChainTracker. ```ts -constructor(network: "main" | "test" | "stn" = "main", config: WhatsOnChainConfig = {}) +constructor(network: "main" | "test" | "stn" = "main", config: WhatsOnChainConfig = {}) ``` + See also: [WhatsOnChainConfig](./transaction.md#interface-whatsonchainconfig) Argument Details -+ **network** - + The BSV network to use when calling the WhatsOnChain API. -+ **config** - + Configuration options for the WhatsOnChain ChainTracker. +- **network** + - The BSV network to use when calling the WhatsOnChain API. +- **config** + - Configuration options for the WhatsOnChain ChainTracker. Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ## Functions -| | -| --- | -| [defaultBroadcaster](#function-defaultbroadcaster) | -| [defaultChainTracker](#function-defaultchaintracker) | -| [defaultHttpClient](#function-defaulthttpclient) | -| [executeNodejsRequest](#function-executenodejsrequest) | -| [isBroadcastFailure](#function-isbroadcastfailure) | -| [isBroadcastResponse](#function-isbroadcastresponse) | +| | | +| ------------------------------------------------------------ | ------------------------------------------------------------------------------------ | +| [assertEvidenceUnchanged](#function-assertevidenceunchanged) | [isBroadcastResponse](#function-isbroadcastresponse) | +| [defaultBroadcaster](#function-defaultbroadcaster) | [parseEvidence](#function-parseevidence) | +| [defaultChainTracker](#function-defaultchaintracker) | [registerScriptVerificationBackend](#function-registerscriptverificationbackend) | +| [defaultHttpClient](#function-defaulthttpclient) | [scopedScriptBackend](#function-scopedscriptbackend) | +| [evidenceError](#function-evidenceerror) | [scriptVerificationBackend](#function-scriptverificationbackend) | +| [evidenceScriptScope](#function-evidencescriptscope) | [unregisterScriptVerificationBackend](#function-unregisterscriptverificationbackend) | +| [executeNodejsRequest](#function-executenodejsrequest) | [withEvidenceScriptWork](#function-withevidencescriptwork) | +| [isBroadcastFailure](#function-isbroadcastfailure) | | + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Function: assertEvidenceUnchanged + +Fence all owned transaction/proof bytes, including backend readiness callbacks. + +```ts +export function assertEvidenceUnchanged(candidate: EvidenceCandidate): void +``` + +See also: [EvidenceCandidate](./transaction.md#interface-evidencecandidate) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -2449,7 +3086,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Function: defaultBroadcaster ```ts -export function defaultBroadcaster(isTestnet: boolean = false, config: ArcConfig = {}): Broadcaster +export function defaultBroadcaster(isTestnet: boolean = false, config: ArcConfig = {}): Broadcaster ``` See also: [ArcConfig](./transaction.md#interface-arcconfig), [Broadcaster](./transaction.md#interface-broadcaster) @@ -2457,10 +3094,11 @@ See also: [ArcConfig](./transaction.md#interface-arcconfig), [Broadcaster](./tra Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Function: defaultChainTracker ```ts -export function defaultChainTracker(): ChainTracker +export function defaultChainTracker(): ChainTracker ``` See also: [ChainTracker](./transaction.md#interface-chaintracker) @@ -2468,14 +3106,15 @@ See also: [ChainTracker](./transaction.md#interface-chaintracker) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Function: defaultHttpClient Returns a default HttpClient implementation based on the environment that it is run on. -This method will attempt to use `window.fetch` if available (in browser environments). -If running in a Node environment, it falls back to using the Node `https` module +This method will attempt to use `window.fetch` if available (in browser environments), +then `globalThis.fetch` (service workers, Deno, Node 18+), then the Node `https` module. ```ts -export function defaultHttpClient(): HttpClient +export function defaultHttpClient(): HttpClient ``` See also: [HttpClient](./transaction.md#interface-httpclient) @@ -2483,37 +3122,70 @@ See also: [HttpClient](./transaction.md#interface-httpclient) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Function: evidenceError + +Keep bounded-work outcomes distinct without exposing errors containing evidence. + +```ts +export function evidenceError(error: unknown): TransactionEvidenceError +``` + +See also: [TransactionEvidenceError](./transaction.md#class-transactionevidenceerror) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Function: evidenceScriptScope + +```ts +export function evidenceScriptScope(tx: Transaction): EvidenceScriptScope | undefined +``` + +See also: [EvidenceScriptScope](./transaction.md#interface-evidencescriptscope), [Transaction](./transaction.md#class-transaction) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Function: executeNodejsRequest Shared implementation for handling Node.js HTTP requests. Used by both NodejsHttpClient and BinaryNodejsHttpClient. ```ts -export function executeNodejsRequest(https: HttpsModuleLike, url: string, requestOptions: HttpClientRequestOptions, serializeData: (data: any) => any): Promise +export function executeNodejsRequest( + https: HttpsModuleLike, + url: string, + requestOptions: HttpClientRequestOptions, + serializeData: (data: any) => any +): Promise ``` See also: [HttpClientRequestOptions](./transaction.md#interface-httpclientrequestoptions), [HttpClientResponse](./transaction.md#type-httpclientresponse), [HttpsModuleLike](./transaction.md#interface-httpsmodulelike) Argument Details -+ **https** - + The Node.js https module (or compatible) -+ **url** - + The URL to make the request to -+ **requestOptions** - + The request configuration -+ **serializeData** - + Function to serialize the request data for writing +- **https** + - The Node.js https module (or compatible) +- **url** + - The URL to make the request to +- **requestOptions** + - The request configuration +- **serializeData** + - Function to serialize the request data for writing Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Function: isBroadcastFailure Convenience type guard for response from `Broadcaster.broadcast` ```ts -export function isBroadcastFailure(r: BroadcastResponse | BroadcastFailure): r is BroadcastFailure +export function isBroadcastFailure(r: BroadcastResponse | BroadcastFailure): r is BroadcastFailure ``` See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [BroadcastResponse](./transaction.md#interface-broadcastresponse) @@ -2521,12 +3193,13 @@ See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [Broa Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Function: isBroadcastResponse Convenience type guard for response from `Broadcaster.broadcast` ```ts -export function isBroadcastResponse(r: BroadcastResponse | BroadcastFailure): r is BroadcastResponse +export function isBroadcastResponse(r: BroadcastResponse | BroadcastFailure): r is BroadcastResponse ``` See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [BroadcastResponse](./transaction.md#interface-broadcastresponse) @@ -2534,12 +3207,103 @@ See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [Broa Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Function: parseEvidence + +```ts +export function parseEvidence( + evidence: TransactionEvidence, + limits: TransactionEvidenceLimits +): EvidenceCandidate +``` + +See also: [EvidenceCandidate](./transaction.md#interface-evidencecandidate), [TransactionEvidence](./transaction.md#interface-transactionevidence), [TransactionEvidenceLimits](./transaction.md#interface-transactionevidencelimits) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Function: registerScriptVerificationBackend + +Installs a process/page-wide optional script backend. + +```ts +export function registerScriptVerificationBackend(backend: ScriptVerificationBackend): void +``` + +See also: [ScriptVerificationBackend](./transaction.md#type-scriptverificationbackend) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Function: scopedScriptBackend + +```ts +export function scopedScriptBackend( + scope: EvidenceScriptScope, + backend: BdkVerifierInterface +): BdkVerifierInterface +``` + +See also: [BdkVerifierInterface](./transaction.md#interface-bdkverifierinterface), [EvidenceScriptScope](./transaction.md#interface-evidencescriptscope) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Function: scriptVerificationBackend + +Returns the currently registered optional script backend, if any. + +```ts +export function scriptVerificationBackend(): ScriptVerificationBackend | undefined +``` + +See also: [ScriptVerificationBackend](./transaction.md#type-scriptverificationbackend) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Function: unregisterScriptVerificationBackend + +Removes `backend` if it is still the active optional implementation. + +```ts +export function unregisterScriptVerificationBackend(backend: ScriptVerificationBackend): void +``` + +See also: [ScriptVerificationBackend](./transaction.md#type-scriptverificationbackend) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Function: withEvidenceScriptWork + +```ts +export async function withEvidenceScriptWork( + tx: Transaction, + scope: EvidenceScriptScope, + verify: () => Promise +): Promise +``` + +See also: [EvidenceScriptScope](./transaction.md#interface-evidencescriptscope), [Transaction](./transaction.md#class-transaction), [verify](./compat.md#variable-verify) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ## Types -| | -| --- | -| [Fetch](#type-fetch) | -| [HttpClientResponse](#type-httpclientresponse) | +| | +| ------------------------------------------------------------------ | +| [Fetch](#type-fetch) | +| [HttpClientResponse](#type-httpclientresponse) | +| [ScriptVerificationBackend](#type-scriptverificationbackend) | +| [TransactionEvidenceErrorCode](#type-transactionevidenceerrorcode) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -2560,49 +3324,80 @@ See also: [FetchOptions](./transaction.md#interface-fetchoptions) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Type: HttpClientResponse An interface for the response returned by the request method. ```ts -export type HttpClientResponse = { - data: T; - status: number; - statusText: string; - ok: true; -} | { - data: any; - status: number; - statusText: string; - ok: false; -} +export type HttpClientResponse = + | { + data: T + status: number + statusText: string + ok: true + } + | { + data: any + status: number + statusText: string + ok: false + } ``` Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Type: ScriptVerificationBackend + +Backend shape shared by transaction-graph and individual-Spend routing. + +```ts +export type ScriptVerificationBackend = BdkVerifierInterface & SpendVerifierInterface +``` + +See also: [BdkVerifierInterface](./transaction.md#interface-bdkverifierinterface), [SpendVerifierInterface](./script.md#interface-spendverifierinterface) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + +### Type: TransactionEvidenceErrorCode + +```ts +export type TransactionEvidenceErrorCode = + 'invalid-evidence' | 'limit' | 'cancelled' | 'timeout' | 'context-changed' | 'disposed' +``` + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ## Enums ### Enum: TX_DATA_FORMAT ```ts export enum TX_DATA_FORMAT { - RAWTX = 0, - RAWTX_AND_BUMP_INDEX = 1, - TXID_ONLY = 2 + RAWTX = 0, + RAWTX_AND_BUMP_INDEX = 1, + TXID_ONLY = 2 } ``` Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ## Variables -| | -| --- | -| [ATOMIC_BEEF](#variable-atomic_beef) | -| [BEEF_V1](#variable-beef_v1) | -| [BEEF_V2](#variable-beef_v2) | +| | +| ------------------------------------------------------------------------------ | +| [ATOMIC_BEEF](#variable-atomic_beef) | +| [BEEF_V1](#variable-beef_v1) | +| [BEEF_V2](#variable-beef_v2) | +| [defaultTransactionEvidenceLimits](#variable-defaulttransactionevidencelimits) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -2617,6 +3412,7 @@ ATOMIC_BEEF = 16843009 Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Variable: BEEF_V1 ```ts @@ -2626,6 +3422,7 @@ BEEF_V1 = 4022206465 Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + ### Variable: BEEF_V2 ```ts @@ -2635,3 +3432,31 @@ BEEF_V2 = 4022206466 Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- + +### Variable: defaultTransactionEvidenceLimits + +```ts +defaultTransactionEvidenceLimits: Readonly = Object.freeze({ + candidateBytes: 1024 * 1024, + retainedBytes: 16 * 1024 * 1024, + transactions: 256, + inputs: 4096, + scriptBytes: 256 * 1024, + scriptMemoryBytes: 16 * 1024 * 1024, + candidatesPerTransaction: 8, + pendingTransactions: 32, + concurrentTransactions: 4, + pendingChainCalls: 8, + consumers: 128, + cacheEntries: 128, + cacheAgeMs: 60000, + attemptTimeoutMs: 5000, + requestTimeoutMs: 15000 +}) +``` + +See also: [TransactionEvidenceLimits](./transaction.md#interface-transactionevidencelimits) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- diff --git a/packages/sdk/docs/transaction-evidence.md b/packages/sdk/docs/transaction-evidence.md new file mode 100644 index 000000000..238bfa4fb --- /dev/null +++ b/packages/sdk/docs/transaction-evidence.md @@ -0,0 +1,160 @@ +# Transaction evidence + +This guide documents the bounded transaction-evidence APIs exported by the SDK. +The C02/C03 implementation is under review; these notes describe the current +source contract and its limits. They do not claim that the wider verification +plan or release gates have completed. + +## Evidence and verification + +The transaction entry point exports `TransactionEvidence`, +`VerifiedTransactionOutput`, `TransactionEvidenceError`, +`TransactionEvidenceLimits`, `defaultTransactionEvidenceLimits`, and +`TransactionEvidenceCoordinator`. The source contracts are in +[`TransactionEvidence.ts`](../src/transaction/TransactionEvidence.ts), +[`TransactionEvidenceCoordinator.ts`](../src/transaction/TransactionEvidenceCoordinator.ts), +and [`ChainTracker.ts`](../src/transaction/ChainTracker.ts). + +`TransactionEvidence` is an untrusted receipt: + +```ts +interface TransactionEvidence { + beef: number[] + outputIndex: number + txid?: string +} +``` + +The optional `txid` is a consistency hint. The coordinator copies the BEEF +bytes before parsing or deriving a receipt, derives the selected transaction ID +from the atomic transaction (or the final transaction when no atomic ID is +present), and rejects a mismatched hint. A successful verification returns the +derived transaction ID, selected output index, canonical outpoint, and locking +script. It establishes transaction inclusion/ancestry and script validity +under the supplied verifier and chain tracker. It does not establish service +relevance, current unspentness, or ownership. + +The parser walks the complete unconfirmed graph on every attempt, including +positive-cache use. It checks source transaction presence, source TXID +consistency, duplicate spent outpoints, input and serialized-script limits, +and rejects an unconfirmed zero-input leaf. Confirmed ancestors still require +canonical Merkle-root validation through the caller's tracker. Receipt bytes +are copied and owned by the SDK; host metadata is never chain authority. + +## Caller context and bounded policy + +The coordinator requires a caller-owned `chainTracker`, nonempty +`chainNamespace`, and nonempty `policyId`. The namespace identifies the +caller's network or application trust domain. The policy ID identifies the +semantic verification policy or backend. Neither is populated from lookup +metadata. + +`ChainTracker.getVerificationContext?: () => string | number` is an optional +synchronous provider, policy, or recovery-generation marker. The optional +`getVerificationContextToken(signal?)` asynchronously obtains a fresh trusted +tip/context token. The coordinator brackets canonical-root and height checks +with that token and rejects a `context-changed` result when the token changes. +Chaintracks and local adapters include monotonic reorganization/reset epochs +and fence reset entry and failure paths. A remote tip read can still miss an +unobserved ABA transition back to the same tip, and a token over multiple +sources is not an atomic multi-source snapshot. Canonical roots and observed +heights are therefore still rechecked on every use, including positive-cache +reuse. The required tracker methods accept optional abort signals; existing +implementations may ignore them. + +The limits in [`TransactionEvidence.ts`](../src/transaction/TransactionEvidence.ts) +are local admission and memory policy, not consensus limits. Defaults are: + +| Limit | Default | +| -------------------------- | ---------: | +| Candidate bytes | 1 MiB | +| Retained evidence bytes | 16 MiB | +| Transactions | 256 | +| Inputs | 4,096 | +| Serialized script bytes | 256 KiB | +| Script memory | 16 MiB | +| Candidates per transaction | 8 | +| Pending transactions | 32 | +| Concurrent transactions | 4 | +| Pending chain calls | 8 | +| Consumers | 128 | +| Positive cache entries | 128 | +| Positive cache age | 60 seconds | +| Attempt timeout | 5 seconds | +| Request timeout | 15 seconds | + +Overrides must be positive safe integers. Invalid limits fail closed with +`TransactionEvidenceError` code `limit`. These limits are configurable local +policy, not consensus limits. Script resource exhaustion is also mapped to the +typed `limit` outcome; malformed or inconsistent evidence remains +`invalid-evidence`. + +The coordinator shares bounded transaction work and canonical chain calls +among consumers. Consumer cancellation detaches that consumer. Non-abortable +script or backend work remains counted until its actual promise settles, and +one consumer cannot cancel another consumer's owned work. Positive results are +bounded by entry count, retained bytes, and age. Rejected receipts are evicted +and do not become permanent negative decisions for a transaction ID. + +An internal `EvidenceScriptWork` hook is now present in +[`EvidenceScriptWork.ts`](../src/transaction/EvidenceScriptWork.ts). Its binding +covers the actual transaction, every input's source transaction/output bytes +and value, and the script policy/backend parameters. The binding is +transaction-wide; it is not an input-index-only key. A hit may skip +cryptographic script execution only. It must still perform source/value +binding, complete graph traversal, canonical chain calls, and policy/context +checks. Non-batch individual executions keep their own fulfilled or rejected +result, including the typed error; only a backend batch-level failure rejects +every in-flight entry. The cache and in-flight owners remain bounded by the +coordinator limits. A normal block arrival before the next reuse does not +erase exact script work when the transaction/source/policy binding remains +valid; canonical anchors and fresh context are still checked. + +## Resolver evidence intake + +`LookupQueryOptions.onEvidence` and `LookupEvidenceEvent` are additive optional +APIs in [`LookupResolver.ts`](../src/overlay-tools/LookupResolver.ts). The +callback receives copied, untrusted per-host receipts before legacy TXID and +outpoint aggregation: + +```ts +type LookupEvidenceEvent = + { type: 'output'; host: string; output: LookupAnswer['outputs'][number] } | { type: 'limit' } +``` + +Intake defaults to 512 outputs or 16 MiB per query and reports one `limit` +event. `evidenceLimits: { maxOutputs?, maxBytes? }` can configure those local +callback bounds; values must be positive safe integers. Coordinate them with +the downstream verifier's admission limits when admitting larger valid +evidence. +Callback completion is not awaited, callback failures are isolated, and no +callback is delivered after the query iterator closes. The legacy answer, +host scheduling, reputation behavior, timeout behavior, and existing +positional call forms remain unchanged. The additive callback preserves the +existing 2-second lookup default and 5-second tracker wait bound. Older or +custom resolvers can continue through the legacy aggregated-answer fallback. + +Aggregated output may carry a host-supplied TXID hint. The resolver uses a +nonempty hint as a fast path and otherwise derives a TXID from BEEF with a +bounded memo. Security-sensitive consumers must still validate the bytes and +compare the derived TXID. Aggregation remains first-wins for each TXID/output +index key; use `onEvidence` when every bounded host receipt is needed. + +## Compatibility + +Existing two-method `ChainTracker` implementations and existing resolver +callers remain source-compatible. The new transaction exports, optional +tracker context and abort signals, and optional resolver callback are additive. +No caller migration is required for existing calls. The SDK's declared source +version is 2.5.0; the root release review owns final release evidence. + +The synthetic shared-ancestor fixture is 556 BEEF bytes with three reachable +transactions, three inputs, and 314 serialized script bytes. Two child graphs +share the signed ancestor and result in three actual script executions. This +is characterization evidence for graph traversal and script-work sharing, not +a general performance or throughput claim. + +No worker, latency, throughput, browser/mobile deployment, C04/C05, or whole +plan completion claim is made here. Run the package's declared build, +typecheck, lint, format, test, packed-consumer, browser, and documentation +checks before treating this guide as release evidence. diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 089328862..9b804e5c7 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/sdk", - "version": "2.4.2", + "version": "2.5.0", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/sdk/src/overlay-tools/LookupResolver.ts b/packages/sdk/src/overlay-tools/LookupResolver.ts index 66f44f1bc..eac559734 100644 --- a/packages/sdk/src/overlay-tools/LookupResolver.ts +++ b/packages/sdk/src/overlay-tools/LookupResolver.ts @@ -53,6 +53,20 @@ export type LookupFacilitatorAnswer = LookupAnswer | LookupFreeformAnswer * All optional; defaults preserve prior behavior. */ export interface LookupQueryOptions { + /** + * Owned, UNTRUSTED receipts before legacy txid/outpoint deduplication. Enqueue + * promptly; callback completion is not awaited and failures are isolated. + * Intake stops at the configured evidenceLimits, reporting one limit event. + * No callbacks occur after the query iterator closes. Legacy answers, host + * scheduling, timeout and reputation behavior are unchanged. + */ + onEvidence?: (event: LookupEvidenceEvent) => void | Promise + /** + * Callback intake budget, independent of legacy aggregation. Defaults to 512 + * outputs / 16 MiB of BEEF and context bytes. Values must be positive safe + * integers. Coordinate these with a downstream verifier's admission limits. + */ + evidenceLimits?: { maxOutputs?: number; maxBytes?: number } /** * Override the grace window (ms) between the first valid response and the resolution of the query. * Late responders arriving within this window are merged into the result. Default 80 ms. @@ -96,6 +110,10 @@ export interface LookupQueryOptions { correlationId?: string } +/** Additive evidence intake, independent of the legacy aggregated answer. */ +export type LookupEvidenceEvent = + { type: 'output'; host: string; output: LookupAnswer['outputs'][number] } | { type: 'limit' } + /** Info supplied to onUnreachableHost callbacks. */ export interface UnreachableHostInfo { /** Host URL that failed. */ @@ -498,10 +516,7 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { } type LookupQueryEvent = - | { kind: 'answer'; answer: LookupAnswer } - | { kind: 'done' } - | { kind: 'grace' } - | { kind: 'soft' } + { kind: 'answer'; answer: LookupAnswer } | { kind: 'done' } | { kind: 'grace' } | { kind: 'soft' } interface LookupQuerySessionOptions { hostCount: number @@ -509,10 +524,8 @@ interface LookupQuerySessionOptions { softTimeoutMs?: number waitForAllHosts: boolean correlationId?: string - resolveTxId: ( - output: LookupAnswer['outputs'][number], - now: number - ) => string | null + evidenceLimits?: LookupQueryOptions['evidenceLimits'] + resolveTxId: (output: LookupAnswer['outputs'][number], now: number) => string | null } class LookupQuerySession { @@ -526,6 +539,12 @@ class LookupQuerySession { rejectedHosts = 0 freeformHosts = 0 emittedFinal = false + closed = false + private evidenceOutputs = 0 + private evidenceBytes = 0 + private evidenceLimited = false + private readonly maxEvidenceOutputs: number + private readonly maxEvidenceBytes: number private readonly graceMs: number private readonly softTimeoutMs?: number @@ -545,6 +564,14 @@ class LookupQuerySession { private emittedOnce = false constructor(options: LookupQuerySessionOptions) { + this.maxEvidenceOutputs = options.evidenceLimits?.maxOutputs ?? 512 + this.maxEvidenceBytes = options.evidenceLimits?.maxBytes ?? 16 * 1024 * 1024 + if ( + ![this.maxEvidenceOutputs, this.maxEvidenceBytes].every( + value => Number.isSafeInteger(value) && value > 0 + ) + ) + throw new Error('Evidence intake limits must be positive safe integers') this.hostCount = options.hostCount this.graceMs = options.graceMs this.softTimeoutMs = options.softTimeoutMs @@ -570,6 +597,43 @@ class LookupQuerySession { this.push({ kind: 'answer', answer }) } + receiveEvidence( + host: string, + answer: LookupAnswer, + callback: LookupQueryOptions['onEvidence'] + ): void { + if (callback === undefined || this.closed || this.evidenceLimited) return + const deliver = (event: LookupEvidenceEvent): void => { + try { + void Promise.resolve(callback(event)).catch(() => {}) + } catch { + /* consumer isolation */ + } + } + for (const output of answer.outputs) { + const bytes = output.beef.length + (output.context?.length ?? 0) + if ( + this.evidenceOutputs >= this.maxEvidenceOutputs || + this.evidenceBytes + bytes > this.maxEvidenceBytes + ) { + this.evidenceLimited = true + deliver({ type: 'limit' }) + break + } + this.evidenceOutputs++ + this.evidenceBytes += bytes + deliver({ + type: 'output', + host, + output: { + ...output, + beef: output.beef.slice(), + ...(output.context === undefined ? {} : { context: output.context.slice() }) + } + }) + } + } + recordFreeformAnswer(): void { this.freeformHosts++ } @@ -615,9 +679,7 @@ class LookupQuerySession { failedHosts: this.failedHosts, rejectedHosts: this.rejectedHosts, freeformHosts: this.freeformHosts, - ...(this.correlationId !== undefined - ? { correlationId: this.correlationId } - : {}) + ...(this.correlationId !== undefined ? { correlationId: this.correlationId } : {}) } } @@ -634,11 +696,7 @@ class LookupQuerySession { this.graceFired = true } } - if ( - this.graceFired && - added && - (this.emittedOnce || !this.waitForAllHosts) - ) { + if (this.graceFired && added && (this.emittedOnce || !this.waitForAllHosts)) { this.emittedOnce = true return this.snapshot(false) } @@ -663,9 +721,7 @@ class LookupQuerySession { } return { snapshot, - stop: - typeof this.softTimeoutMs === 'number' && - this.firstResponseAt !== null + stop: typeof this.softTimeoutMs === 'number' && this.firstResponseAt !== null } } @@ -695,14 +751,8 @@ class LookupQuerySession { } async *progress(): AsyncIterable { - if ( - typeof this.softTimeoutMs === 'number' && - this.softTimeoutMs >= 0 - ) { - this.softTimer = setTimeout( - () => this.push({ kind: 'soft' }), - this.softTimeoutMs - ) + if (typeof this.softTimeoutMs === 'number' && this.softTimeoutMs >= 0) { + this.softTimer = setTimeout(() => this.push({ kind: 'soft' }), this.softTimeoutMs) } try { let stop = false @@ -716,6 +766,7 @@ class LookupQuerySession { this.emittedFinal = true yield finalSnapshot } finally { + this.closed = true if (this.graceTimer !== null) clearTimeout(this.graceTimer) if (this.softTimer !== null) clearTimeout(this.softTimer) } @@ -889,10 +940,7 @@ export default class LookupResolver { private async competentHostsFor(question: LookupQuestion): Promise { let hosts: string[] if (question.service === 'ls_slap') { - hosts = - this.networkPreset === 'local' - ? ['http://localhost:8080'] - : this.slapTrackers + hosts = this.networkPreset === 'local' ? ['http://localhost:8080'] : this.slapTrackers } else if (this.hostOverrides[question.service] != null) { hosts = this.hostOverrides[question.service] } else if (this.networkPreset === 'local') { @@ -911,9 +959,7 @@ export default class LookupResolver { private isSlapRecoveryEligible(service: string): boolean { return ( - service !== 'ls_slap' && - this.hostOverrides[service] == null && - this.networkPreset !== 'local' + service !== 'ls_slap' && this.hostOverrides[service] == null && this.networkPreset !== 'local' ) } @@ -921,10 +967,7 @@ export default class LookupResolver { const competentHosts = await this.competentHostsFor(question) let rankedHosts: string[] try { - rankedHosts = this.prepareHostsForQuery( - competentHosts, - `lookup service ${question.service}` - ) + rankedHosts = this.prepareHostsForQuery(competentHosts, `lookup service ${question.service}`) } catch (error) { if (!this.isSlapRecoveryEligible(question.service)) throw error this.hostsCache.delete(question.service) @@ -935,10 +978,7 @@ export default class LookupResolver { `No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${question.service}` ) } - rankedHosts = this.prepareHostsForQuery( - fresh, - `lookup service ${question.service}` - ) + rankedHosts = this.prepareHostsForQuery(fresh, `lookup service ${question.service}`) } if (rankedHosts.length < 1) { throw new Error( @@ -948,13 +988,9 @@ export default class LookupResolver { return rankedHosts } - private unreachableNotificationCooldown( - options: LookupQueryOptions | undefined - ): number { + private unreachableNotificationCooldown(options: LookupQueryOptions | undefined): number { const requested = options?.unreachableHostNotificationCooldownMs - return typeof requested === 'number' && - Number.isFinite(requested) && - requested >= 0 + return typeof requested === 'number' && Number.isFinite(requested) && requested >= 0 ? requested : DEFAULT_UNREACHABLE_NOTIFICATION_COOLDOWN_MS } @@ -970,13 +1006,9 @@ export default class LookupResolver { const notificationKey = `${service}\u0000${host}` const now = Date.now() const lastNotificationAt = - this.lastUnreachableNotificationAt.get(notificationKey) ?? - Number.NEGATIVE_INFINITY + this.lastUnreachableNotificationAt.get(notificationKey) ?? Number.NEGATIVE_INFINITY if (now - lastNotificationAt < cooldownMs) return - if ( - this.lastUnreachableNotificationAt.size >= - MAX_NOTIFICATION_DEDUP_ENTRIES - ) { + if (this.lastUnreachableNotificationAt.size >= MAX_NOTIFICATION_DEDUP_ENTRIES) { this.evictOldest(this.lastUnreachableNotificationAt) } this.lastUnreachableNotificationAt.set(notificationKey, now) @@ -1015,19 +1047,10 @@ export default class LookupResolver { return } session.recordFreeformAnswer() - this.captureHostTelemetry( - service, - host, - 'freeform', - Date.now() - hostStartedAt, - correlationId - ) + this.captureHostTelemetry(service, host, 'freeform', Date.now() - hostStartedAt, correlationId) } - private recordLookupHostFailure( - context: LookupHostFailureContext, - error: unknown - ): void { + private recordLookupHostFailure(context: LookupHostFailureContext, error: unknown): void { const { session, service, @@ -1049,13 +1072,7 @@ export default class LookupResolver { error ) if (!semanticRejection) { - this.notifyUnreachableHost( - host, - service, - error, - onUnreachableHost, - notificationCooldownMs - ) + this.notifyUnreachableHost(host, service, error, onUnreachableHost, notificationCooldownMs) } } @@ -1067,12 +1084,12 @@ export default class LookupResolver { options: LookupQueryOptions | undefined ): void { const correlationId = session.correlationId - const notificationCooldownMs = - this.unreachableNotificationCooldown(options) + const notificationCooldownMs = this.unreachableNotificationCooldown(options) for (const host of hosts) { const hostStartedAt = Date.now() void this.lookupHostWithTracking(host, question, timeout) .then(answer => { + if (isOutputListAnswer(answer)) session.receiveEvidence(host, answer, options?.onEvidence) this.recordLookupHostAnswer( session, question.service, @@ -1083,15 +1100,18 @@ export default class LookupResolver { ) }) .catch(error => { - this.recordLookupHostFailure({ - session, - service: question.service, - host, - hostStartedAt, - correlationId, - onUnreachableHost: options?.onUnreachableHost, - notificationCooldownMs - }, error) + this.recordLookupHostFailure( + { + session, + service: question.service, + host, + hostStartedAt, + correlationId, + onUnreachableHost: options?.onUnreachableHost, + notificationCooldownMs + }, + error + ) }) .finally(() => { session.recordDone() @@ -1123,11 +1143,11 @@ export default class LookupResolver { options?.correlationId ?? (this.telemetry.enabled ? this.telemetry.createCorrelationId() : undefined) const session = new LookupQuerySession({ + evidenceLimits: options?.evidenceLimits, hostCount, graceMs: options?.graceMs ?? 80, softTimeoutMs: options?.softTimeoutMs, - waitForAllHosts: - options?.waitForAllHosts ?? options?.holdForUnknownHosts ?? false, + waitForAllHosts: options?.waitForAllHosts ?? options?.holdForUnknownHosts ?? false, correlationId, resolveTxId: (output, now) => this.resolveTxIdForOutput(output, now) }) @@ -1144,13 +1164,7 @@ export default class LookupResolver { } }) - this.startLookupHostQueries( - rankedHosts, - question, - timeout, - session, - options - ) + this.startLookupHostQueries(rankedHosts, question, timeout, session, options) try { for await (const progress of session.progress()) { diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.evidence.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.evidence.test.ts new file mode 100644 index 000000000..6c10ebcb6 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.evidence.test.ts @@ -0,0 +1,232 @@ +import LookupResolver, { type LookupFacilitatorAnswer } from '../LookupResolver' +import MerklePath from '../../transaction/MerklePath' +import Transaction from '../../transaction/Transaction' +import P2PKH from '../../script/templates/P2PKH' +import PrivateKey from '../../primitives/PrivateKey' +import Script from '../../script/Script' + +const service = 'ls_identity' + +interface Deferred { + promise: Promise + resolve: (value: T) => void +} + +function deferred(): Deferred { + let resolve!: (value: T) => void + const promise = new Promise(resolvePromise => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +async function sameTransactionReceipts(): Promise<{ bad: number[]; valid: number[] }> { + const key = new PrivateKey(42) + const p2pkh = new P2PKH() + const source = new Transaction() + source.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + source.addOutput({ satoshis: 10, lockingScript: p2pkh.lock(key.toAddress()) }) + source.merklePath = new MerklePath(700_000, [ + [ + { offset: 0, hash: source.id('hex'), txid: true }, + { offset: 1, duplicate: true } + ] + ]) + + const tx = new Transaction() + tx.addInput({ + sourceTransaction: source, + sourceOutputIndex: 0, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + tx.addOutput({ satoshis: 4, lockingScript: p2pkh.lock(key.toAddress()) }) + await tx.sign() + const bad = tx.toBEEF() + + const alternate = Transaction.fromBEEF(bad) + const alternateSource = alternate.inputs[0].sourceTransaction + if (alternateSource === undefined) throw new Error('fixture source is missing') + alternateSource.merklePath = new MerklePath(700_000, [ + [ + { offset: 0, hash: alternateSource.id('hex'), txid: true }, + { offset: 1, hash: '42'.repeat(32) } + ] + ]) + const valid = alternate.toBEEF() + expect(Transaction.fromBEEF(bad).id('hex')).toBe(Transaction.fromBEEF(valid).id('hex')) + return { bad, valid } +} + +function resolverFor( + hosts: string[], + lookup: (host: string) => Promise +): LookupResolver { + return new LookupResolver({ + hostOverrides: { [service]: hosts }, + facilitator: { lookup: async host => await lookup(host) } + }) +} + +describe('LookupResolver additive evidence intake', () => { + it('delivers both owned receipts before legacy first-wins aggregation and isolates callback mutations', async () => { + const { bad, valid } = await sameTransactionReceipts() + const firstHost = 'https://first.invalid-proof.example' + const secondHost = 'https://second.valid-proof.example' + const callbackEvents: Array<{ host: string; beef: number[] }> = [] + const responses: Record = { + [firstHost]: { type: 'output-list', outputs: [{ beef: bad, outputIndex: 0, context: [1] }] }, + [secondHost]: { + type: 'output-list', + outputs: [{ beef: valid, outputIndex: 0, context: [2] }] + } + } + const resolver = resolverFor([firstHost, secondHost], async host => responses[host]) + + const answer = await resolver.query({ service, query: {} }, undefined, { + onEvidence: event => { + if (event.type !== 'output') return + callbackEvents.push({ host: event.host, beef: event.output.beef.slice() }) + event.output.beef.fill(0) + event.output.context?.fill(0) + } + }) + + expect(callbackEvents).toEqual([ + { host: firstHost, beef: bad }, + { host: secondHost, beef: valid } + ]) + expect(answer.outputs).toEqual([{ beef: bad, outputIndex: 0, context: [1] }]) + expect(responses[firstHost]).toEqual({ + type: 'output-list', + outputs: [{ beef: bad, outputIndex: 0, context: [1] }] + }) + expect(responses[secondHost]).toEqual({ + type: 'output-list', + outputs: [{ beef: valid, outputIndex: 0, context: [2] }] + }) + }) + + it('does not emit untrusted receipts for malformed, empty, or freeform answers', async () => { + const malformedHost = 'https://malformed.example' + const emptyHost = 'https://empty.example' + const freeformHost = 'https://freeform.example' + const resolver = resolverFor([malformedHost, emptyHost, freeformHost], async host => { + if (host === malformedHost) + return { type: 'output-list', outputs: [{ beef: [], outputIndex: 0 }] } as never + if (host === emptyHost) return { type: 'output-list', outputs: [] } + return { type: 'freeform', result: { untrusted: true } } + }) + const received: unknown[] = [] + + await expect( + resolver.query({ service, query: {} }, undefined, { + onEvidence: event => received.push(event) + }) + ).resolves.toEqual({ + type: 'output-list', + outputs: [] + }) + expect(received).toEqual([]) + }) + + it('isolates a synchronous callback failure from the legacy answer', async () => { + const { bad } = await sameTransactionReceipts() + const resolver = resolverFor(['https://callback-throws.example'], async () => ({ + type: 'output-list', + outputs: [{ beef: bad, outputIndex: 0 }] + })) + + await expect( + resolver.query({ service, query: {} }, undefined, { + onEvidence: () => { + throw new Error('consumer failure') + } + }) + ).resolves.toEqual({ type: 'output-list', outputs: [{ beef: bad, outputIndex: 0 }] }) + }) + + it('closes evidence delivery when a progressive iterator is closed before a late host responds', async () => { + const { bad, valid } = await sameTransactionReceipts() + const firstHost = 'https://fast.example' + const lateHost = 'https://late.example' + const late = deferred() + const events: string[] = [] + const resolver = resolverFor([firstHost, lateHost], async host => + host === firstHost + ? { type: 'output-list', outputs: [{ beef: bad, outputIndex: 0 }] } + : await late.promise + ) + const iterator = resolver + .query$({ service, query: {} }, undefined, { + graceMs: 0, + onEvidence: event => { + if (event.type === 'output') events.push(event.host) + } + }) + [Symbol.asyncIterator]() + + await expect(iterator.next()).resolves.toMatchObject({ done: false }) + await iterator.return?.() + late.resolve({ type: 'output-list', outputs: [{ beef: valid, outputIndex: 0 }] }) + await new Promise(resolve => setImmediate(resolve)) + expect(events).toEqual([firstHost]) + }) + + it('reports each bounded intake limit once while preserving the legacy aggregation', async () => { + const { bad } = await sameTransactionReceipts() + const manyOutputs = Array.from({ length: 513 }, () => ({ beef: bad.slice(), outputIndex: 0 })) + const countEvents: Array<'output' | 'limit'> = [] + const countResolver = resolverFor(['https://count-limit.example'], async () => ({ + type: 'output-list', + outputs: manyOutputs + })) + + const counted = await countResolver.query({ service, query: {} }, undefined, { + onEvidence: event => countEvents.push(event.type) + }) + expect(countEvents.filter(type => type === 'output')).toHaveLength(512) + expect(countEvents.filter(type => type === 'limit')).toHaveLength(1) + expect(counted.outputs).toEqual([{ beef: bad, outputIndex: 0 }]) + + const bytesEvents: Array<'output' | 'limit'> = [] + const tooLarge = Array.from({ length: 16 * 1024 * 1024 + 1 }, () => 0) + const bytesResolver = resolverFor(['https://bytes-limit.example'], async () => ({ + type: 'output-list', + outputs: [{ beef: tooLarge, outputIndex: 0 }] + })) + await bytesResolver.query({ service, query: {} }, undefined, { + onEvidence: event => bytesEvents.push(event.type) + }) + expect(bytesEvents).toEqual(['limit']) + }) + + it('applies caller-configured evidence limits without changing legacy aggregation', async () => { + const { bad, valid } = await sameTransactionReceipts() + const outputs = [ + { beef: bad, outputIndex: 0 }, + { beef: valid, outputIndex: 0 } + ] + const resolver = resolverFor(['https://configured-limits.example'], async () => ({ + type: 'output-list', + outputs + })) + const limited: Array<'output' | 'limit'> = [] + const answer = await resolver.query({ service, query: {} }, undefined, { + evidenceLimits: { maxOutputs: 1, maxBytes: bad.length * 2 }, + onEvidence: event => limited.push(event.type) + }) + expect(limited).toEqual(['output', 'limit']) + expect(answer.outputs).toEqual([{ beef: bad, outputIndex: 0 }]) + + const admitted: Array<'output' | 'limit'> = [] + await resolver.query({ service, query: {} }, undefined, { + evidenceLimits: { maxOutputs: 2, maxBytes: bad.length + valid.length }, + onEvidence: event => admitted.push(event.type) + }) + expect(admitted).toEqual(['output', 'output']) + }) +}) diff --git a/packages/sdk/src/transaction/ChainTracker.ts b/packages/sdk/src/transaction/ChainTracker.ts index 20552f847..9d2d78f12 100644 --- a/packages/sdk/src/transaction/ChainTracker.ts +++ b/packages/sdk/src/transaction/ChainTracker.ts @@ -23,6 +23,21 @@ * }; */ export default interface ChainTracker { - isValidRootForHeight: (root: string, height: number) => Promise - currentHeight: () => Promise + isValidRootForHeight: (root: string, height: number, signal?: AbortSignal) => Promise + currentHeight: (signal?: AbortSignal) => Promise + /** + * Optional trusted local provider/policy/recovery context. Change this value + * when switching sources or resetting their state. It is not a canonical + * chain snapshot: consumers must still check current canonical dependencies. + * Implementations without cancellable I/O may ignore the optional signals. + */ + getVerificationContext?: () => string | number + /** + * Optional fresh canonical context token from the trusted chain provider. + * Include canonical block/tip identity and any available monotonic reorg or + * reset epoch. Consumers compare tokens around asynchronous verification. + * Two remote tip observations are not an atomic snapshot and cannot detect + * an intervening transition back to the identical tip (ABA). + */ + getVerificationContextToken?: (signal?: AbortSignal) => Promise } diff --git a/packages/sdk/src/transaction/EvidenceScriptWork.ts b/packages/sdk/src/transaction/EvidenceScriptWork.ts new file mode 100644 index 000000000..4c6d37c91 --- /dev/null +++ b/packages/sdk/src/transaction/EvidenceScriptWork.ts @@ -0,0 +1,291 @@ +import type Transaction from './Transaction.js' +import type BdkVerifierInterface from './BdkVerifierInterface.js' +import type { BdkVerifyScriptsParams } from './BdkVerifierInterface.js' +import { sha256 } from '../primitives/Hash.js' +import { toHex, toArray } from '../primitives/utils.js' +import { + evidenceError, + TransactionEvidenceError, + type TransactionEvidenceLimits +} from './TransactionEvidence.js' + +interface InputResult { + valid: boolean + inputTotal: number +} +interface Entry { + key: string + generation: number + owners: Set + promise: Promise + resolve: (valid: boolean) => void + reject: (error: unknown) => void + params: BdkVerifyScriptsParams +} + +/** Binds exact script execution inputs; never an ancestor/chain verdict. */ +function binding(params: BdkVerifyScriptsParams): string { + const tx = params.tx + const sources = tx.inputs.map(input => { + const source = input.sourceTransaction + const output = source?.outputs[input.sourceOutputIndex] + if (source === undefined || output === undefined) + throw new TransactionEvidenceError('invalid-evidence') + return [ + source.id('hex'), + input.sourceOutputIndex, + output.satoshis, + output.lockingScript.toHex() + ] + }) + return toHex( + sha256( + toArray( + JSON.stringify([ + tx.toHex(), + sources, + params.blockHeight, + params.consensus, + params.verifyFlags, + params.memoryLimit + ]), + 'utf8' + ) + ) + ) +} + +/** Internal cache. Only coordinator-owned transactions are bound to its scope below. */ +export class EvidenceScriptWork { + private readonly positives = new Map() + private readonly pending = new Map() + private outstanding = 0 + private generation = 0 + private expiryTimer?: ReturnType + + constructor(private readonly limits: Readonly) {} + + clear(): void { + this.generation++ + this.positives.clear() + // Running non-abortable calls remain charged until their actual completion. + this.pending.clear() + clearTimeout(this.expiryTimer) + } + + private has(key: string): boolean { + const expires = this.positives.get(key) + if (expires !== undefined && expires > Date.now()) return true + this.positives.delete(key) + return false + } + + private remember(key: string): void { + if (this.positives.size >= this.limits.cacheEntries) + this.positives.delete(this.positives.keys().next().value!) + this.positives.set(key, Date.now() + this.limits.cacheAgeMs) + this.scheduleExpiry() + } + + private scheduleExpiry(): void { + clearTimeout(this.expiryTimer) + const expiresAt = Math.min(...this.positives.values()) + if (!Number.isFinite(expiresAt)) return + this.expiryTimer = setTimeout( + () => { + for (const key of this.positives.keys()) this.has(key) + this.scheduleExpiry() + }, + Math.max(1, expiresAt - Date.now()) + ) + this.expiryTimer.unref?.() + } + + inputs( + scope: EvidenceScriptScope, + params: BdkVerifyScriptsParams, + verify: (skipScripts: boolean) => InputResult + ): InputResult { + scope.check() + const key = binding(params) + const hit = this.has(key) + // Even a hit traverses every source, queues ancestry and recomputes input value. + const result = verify(hit) + scope.check() + if (binding(params) !== key) throw new TransactionEvidenceError('invalid-evidence') + if (result.valid && !hit) this.remember(key) + return result + } + + async batch( + scope: EvidenceScriptScope, + params: readonly BdkVerifyScriptsParams[], + backend: BdkVerifierInterface + ): Promise { + scope.check() + const created: Entry[] = [] + const results = params.map(param => { + const key = binding(param) + if (this.has(key)) + return Promise.resolve().then(() => { + scope.check() + if (binding(param) !== key) throw new TransactionEvidenceError('invalid-evidence') + return true + }) + let entry = this.pending.get(key) + if (entry === undefined) { + if (this.outstanding >= this.limits.transactions) + return Promise.reject(new TransactionEvidenceError('limit')) + let resolve: Entry['resolve'] = () => {} + let reject: Entry['reject'] = () => {} + const promise = new Promise((accept, fail) => { + resolve = accept + reject = fail + }) + entry = { + key, + generation: this.generation, + owners: new Set(), + promise, + resolve, + reject, + params: param + } + this.outstanding++ + this.pending.set(key, entry) + created.push(entry) + } + const owned = entry + owned.owners.add(scope) + const abandon = (): void => { + owned.owners.delete(scope) + if (owned.owners.size === 0 && this.pending.get(key) === owned) this.pending.delete(key) + } + scope.signal.addEventListener('abort', abandon, { once: true }) + // The outer attempt handles cancellation. Keep non-abortable backend work + // countable until actual settlement; one owner cannot cancel another. + return owned.promise + .then(valid => { + scope.check() + if (binding(param) !== key) throw new TransactionEvidenceError('invalid-evidence') + return valid === true + }) + .finally(() => { + scope.signal.removeEventListener('abort', abandon) + abandon() + }) + }) + if (created.length > 0) void this.execute(created, backend) + const settled = await Promise.allSettled(results) + const values = settled.map(value => { + if (value.status === 'rejected') { + throw evidenceError(value.reason) + } + return value.value + }) + scope.check() + return values + } + + private async execute(entries: Entry[], backend: BdkVerifierInterface): Promise { + try { + const params = entries.map(entry => entry.params) + if (backend.verifyScriptsBatch === undefined) { + const settled = await Promise.allSettled( + params.map(async param => await backend.verifyScripts(param)) + ) + entries.forEach((entry, index) => { + this.settleEntry(entry, settled[index]) + }) + return + } + const values = await backend.verifyScriptsBatch(params) + if (values.length !== entries.length) throw new TransactionEvidenceError('invalid-evidence') + entries.forEach((entry, index) => { + this.publishEntry(entry, values[index] === true) + }) + } catch (error) { + for (const entry of entries) entry.reject(evidenceError(error)) + } finally { + for (const entry of entries) { + this.outstanding-- + if (this.pending.get(entry.key) === entry) this.pending.delete(entry.key) + } + } + } + + private settleEntry(entry: Entry, result: PromiseSettledResult | undefined): void { + if (result === undefined || result.status === 'rejected') { + entry.reject(evidenceError(result?.reason)) + return + } + this.publishEntry(entry, result.value === true) + } + + private publishEntry(entry: Entry, valid: boolean): void { + if (binding(entry.params) !== entry.key) { + entry.reject(new TransactionEvidenceError('invalid-evidence')) + return + } + let live = false + for (const owner of entry.owners) { + try { + owner.check() + live = true + } catch { + /* stale ownership cannot publish */ + } + } + if (valid && live && entry.generation === this.generation) this.remember(entry.key) + entry.resolve(valid) + } +} + +export interface EvidenceScriptScope { + work: EvidenceScriptWork + signal: AbortSignal + check: () => void +} + +// No host metadata, Transaction field or serialized flag can construct this +// association. Its owner is the coordinator's byte-snapshotted candidate. +const scopes = new WeakMap() + +export async function withEvidenceScriptWork( + tx: Transaction, + scope: EvidenceScriptScope, + verify: () => Promise +): Promise { + scopes.set(tx, scope) + try { + return await verify() + } finally { + scopes.delete(tx) + } +} + +export function evidenceScriptScope(tx: Transaction): EvidenceScriptScope | undefined { + return scopes.get(tx) +} + +export function scopedScriptBackend( + scope: EvidenceScriptScope, + backend: BdkVerifierInterface +): BdkVerifierInterface { + return { + supportsMemoryLimit: backend.supportsMemoryLimit, + shouldVerifyScripts: + backend.shouldVerifyScripts === undefined + ? undefined + : params => { + scope.check() + const key = binding(params) + const ready = backend.shouldVerifyScripts!(params) + scope.check() + if (binding(params) !== key) throw new TransactionEvidenceError('invalid-evidence') + return ready + }, + verifyScripts: async params => (await scope.work.batch(scope, [params], backend))[0], + verifyScriptsBatch: async params => await scope.work.batch(scope, params, backend) + } +} diff --git a/packages/sdk/src/transaction/Transaction.ts b/packages/sdk/src/transaction/Transaction.ts index b50d9f633..5370211f5 100644 --- a/packages/sdk/src/transaction/Transaction.ts +++ b/packages/sdk/src/transaction/Transaction.ts @@ -34,6 +34,11 @@ import TransactionSignature, { import Random from '../primitives/Random.js' import type BdkVerifierInterface from './BdkVerifierInterface.js' import { scriptVerificationBackend } from './ScriptVerificationBackend.js' +import { + evidenceScriptScope, + scopedScriptBackend, + type EvidenceScriptScope +} from './EvidenceScriptWork.js' /** Post-Chronicle height used when an input's source UTXO mined-height is unobtainable. */ const POST_CHRONICLE_HEIGHT_FALLBACK = 943816 @@ -59,6 +64,7 @@ type UnminedTransactionVerificationContext = TransactionVerificationState & { feeModel: FeeModel | undefined selectedVerifier: BdkVerifierInterface | undefined verifierQueue: QueuedScriptVerification[] + scriptWork?: EvidenceScriptScope } /** @@ -1141,10 +1147,7 @@ export default class Transaction { } return } - if ( - !state.verifiedTxids.has(sourceTxid) && - !state.queuedTxids.has(sourceTxid) - ) { + if (!state.verifiedTxids.has(sourceTxid) && !state.queuedTxids.has(sourceTxid)) { state.txQueue.push(sourceTransaction) state.queuedTxids.add(sourceTxid) } @@ -1177,11 +1180,7 @@ export default class Transaction { state.scriptsOnly && input.sourceTXID !== undefined ? input.sourceTXID : sourceTransaction.id('hex') - this.queueSourceTransactionForVerification( - sourceTransaction, - sourceTxid, - state - ) + this.queueSourceTransactionForVerification(sourceTransaction, sourceTxid, state) input.sourceTXID ??= sourceTxid if ( !useVerifier && @@ -1229,9 +1228,7 @@ export default class Transaction { const scriptVerdicts = selectedVerifier.verifyScriptsBatch === undefined ? await Promise.all( - verifierQueue.map( - async params => await selectedVerifier.verifyScripts(params) - ) + verifierQueue.map(async params => await selectedVerifier.verifyScripts(params)) ) : await selectedVerifier.verifyScriptsBatch(verifierQueue) if (scriptVerdicts.length !== verifierQueue.length) { @@ -1270,15 +1267,14 @@ export default class Transaction { } as const const useVerifier = selectedVerifier !== undefined && - (memoryLimit === undefined || - selectedVerifier.supportsMemoryLimit === true) && + (memoryLimit === undefined || selectedVerifier.supportsMemoryLimit === true) && (selectedVerifier.shouldVerifyScripts?.(verifierParams) ?? true) - const inputVerification = this.verifyTransactionInputs( - tx, - useVerifier, - getTxid, - context - ) + const verifyInputs = (skipScripts: boolean): { valid: boolean; inputTotal: number } => + this.verifyTransactionInputs(tx, skipScripts, getTxid, context) + const inputVerification = + !useVerifier && context.scriptWork !== undefined + ? context.scriptWork.work.inputs(context.scriptWork, verifierParams, verifyInputs) + : verifyInputs(useVerifier) if (!inputVerification.valid) return false if (useVerifier) verifierQueue.push(verifierParams) if (this.totalVerifiedOutputs(tx) > inputVerification.inputTotal) return false @@ -1308,7 +1304,12 @@ export default class Transaction { verifier?: BdkVerifierInterface ): Promise { const scriptsOnly = chainTracker === 'scripts only' - const selectedVerifier = verifier ?? scriptVerificationBackend() + const backend = verifier ?? scriptVerificationBackend() + const scriptWork = scriptsOnly ? undefined : evidenceScriptScope(this) + const selectedVerifier = + scriptWork !== undefined && backend !== undefined + ? scopedScriptBackend(scriptWork, backend) + : backend if (!scriptsOnly) this.materializeSourceTXIDs() const verifiedTxids = new Set() const verifiedTransactions = new Set() @@ -1327,7 +1328,8 @@ export default class Transaction { verifiedTxids, feeModel, selectedVerifier, - verifierQueue + verifierQueue, + scriptWork } let queueIndex = 0 @@ -1338,13 +1340,7 @@ export default class Transaction { txid ??= tx.id('hex') return txid } - if ( - this.isTransactionAlreadyVerified( - tx, - getTxid, - verificationContext - ) - ) { + if (this.isTransactionAlreadyVerified(tx, getTxid, verificationContext)) { continue } @@ -1360,11 +1356,7 @@ export default class Transaction { ) { continue } - if (!(await this.verifyUnminedTransaction( - tx, - getTxid, - verificationContext - ))) return false + if (!(await this.verifyUnminedTransaction(tx, getTxid, verificationContext))) return false } await this.verifyQueuedScripts(verifierQueue, selectedVerifier) @@ -1403,9 +1395,10 @@ export default class Transaction { } } - private collectBEEFTransactions( - allowPartial?: boolean - ): { bumps: MerklePath[]; txs: Array<{ tx: Transaction; pathIndex?: number }> } { + private collectBEEFTransactions(allowPartial?: boolean): { + bumps: MerklePath[] + txs: Array<{ tx: Transaction; pathIndex?: number }> + } { const bumps: MerklePath[] = [] const bumpIndexByInstance = new Map() const bumpIndexByRoot = new Map() @@ -1418,7 +1411,14 @@ export default class Transaction { const frame = stack.pop() if (frame == null) continue if (frame.expanded) { - this.appendBEEFTransaction(frame.tx, seenTxids, txs, bumps, bumpIndexByInstance, bumpIndexByRoot) + this.appendBEEFTransaction( + frame.tx, + seenTxids, + txs, + bumps, + bumpIndexByInstance, + bumpIndexByRoot + ) continue } this.scheduleBEEFTransaction(frame.tx, allowPartial, scheduledTxids, stack) diff --git a/packages/sdk/src/transaction/TransactionEvidence.ts b/packages/sdk/src/transaction/TransactionEvidence.ts new file mode 100644 index 000000000..070bbea6c --- /dev/null +++ b/packages/sdk/src/transaction/TransactionEvidence.ts @@ -0,0 +1,163 @@ +import { Beef } from './Beef.js' +import type Transaction from './Transaction.js' +import type LockingScript from '../script/LockingScript.js' +import ScriptResourceLimitError from '../script/ScriptResourceLimitError.js' +import { sha256 } from '../primitives/Hash.js' +import { toHex } from '../primitives/utils.js' + +/** Untrusted transaction evidence. The optional txid is only a consistency hint. */ +export interface TransactionEvidence { + beef: number[] + outputIndex: number + txid?: string +} + +/** Verified transaction inclusion/ancestry; no service relevance or unspentness claim. */ +export interface VerifiedTransactionOutput { + readonly txid: string + readonly outputIndex: number + readonly outpoint: string + readonly lockingScript: LockingScript +} + +export type TransactionEvidenceErrorCode = + 'invalid-evidence' | 'limit' | 'cancelled' | 'timeout' | 'context-changed' | 'disposed' + +/** Bounded, payload-free outcome; rejected receipts never permanently reject a txid. */ +export class TransactionEvidenceError extends Error { + constructor(public readonly code: TransactionEvidenceErrorCode) { + super(`Transaction evidence: ${code}`) + this.name = 'TransactionEvidenceError' + } +} + +/** Keep bounded-work outcomes distinct without exposing errors containing evidence. */ +export function evidenceError(error: unknown): TransactionEvidenceError { + if (error instanceof TransactionEvidenceError) return error + return new TransactionEvidenceError( + error instanceof ScriptResourceLimitError ? 'limit' : 'invalid-evidence' + ) +} + +/** Local admission policy, not consensus limits. Byte limits count serialized bytes. */ +export interface TransactionEvidenceLimits { + candidateBytes: number + retainedBytes: number + transactions: number + inputs: number + scriptBytes: number + scriptMemoryBytes: number + candidatesPerTransaction: number + pendingTransactions: number + concurrentTransactions: number + pendingChainCalls: number + consumers: number + cacheEntries: number + cacheAgeMs: number + attemptTimeoutMs: number + requestTimeoutMs: number +} + +export const defaultTransactionEvidenceLimits: Readonly = Object.freeze({ + candidateBytes: 1024 * 1024, + retainedBytes: 16 * 1024 * 1024, + transactions: 256, + inputs: 4096, + scriptBytes: 256 * 1024, + scriptMemoryBytes: 16 * 1024 * 1024, + candidatesPerTransaction: 8, + pendingTransactions: 32, + concurrentTransactions: 4, + pendingChainCalls: 8, + consumers: 128, + cacheEntries: 128, + cacheAgeMs: 60_000, + attemptTimeoutMs: 5000, + requestTimeoutMs: 15_000 +}) + +/** Internal owned candidate. Never constructed from a host's verification assertion. */ +export interface EvidenceCandidate { + tx: Transaction + txid: string + receipt: string + byteLength: number + outputIndex: number + graphBinding: string +} + +export function parseEvidence( + evidence: TransactionEvidence, + limits: TransactionEvidenceLimits +): EvidenceCandidate { + const outputIndex = evidence.outputIndex + const hint = evidence.txid + if (!Array.isArray(evidence.beef) || !Number.isSafeInteger(outputIndex) || outputIndex < 0) { + throw new TransactionEvidenceError('invalid-evidence') + } + if (evidence.beef.length > limits.candidateBytes) throw new TransactionEvidenceError('limit') + // Snapshot and validate BEFORE either parsing or computing the receipt digest. + const bytes = evidence.beef.slice() + if (bytes.length === 0 || bytes.some(byte => !Number.isInteger(byte) || byte < 0 || byte > 255)) { + throw new TransactionEvidenceError('invalid-evidence') + } + const beef = Beef.fromBinary(bytes) + if (beef.txs.length > limits.transactions) throw new TransactionEvidenceError('limit') + const target = beef.atomicTxid ?? beef.txs.at(-1)?.txid + const tx = target === undefined ? undefined : beef.findAtomicTransaction(target) + if (tx === undefined) throw new TransactionEvidenceError('invalid-evidence') + const txid = tx.id('hex') + if (hint !== undefined && (typeof hint !== 'string' || hint.toLowerCase() !== txid)) { + throw new TransactionEvidenceError('invalid-evidence') + } + if (outputIndex >= tx.outputs.length) throw new TransactionEvidenceError('invalid-evidence') + + // Always walk the COMPLETE unconfirmed graph, including on a positive cache hit. + // A per-ancestor verdict cannot establish graph-wide spend consistency. + const pending = [tx] + const visited = new Set() + const spent = new Set() + let inputs = 0 + let scriptBytes = 0 + while (pending.length > 0) { + const current = pending.pop()! + const id = current.id('hex') + if (visited.has(id)) continue + visited.add(id) + for (const output of current.outputs) scriptBytes += output.lockingScript.toBinary().length + for (const input of current.inputs) scriptBytes += input.unlockingScript?.toBinary().length ?? 0 + inputs += current.inputs.length + if (inputs > limits.inputs || scriptBytes > limits.scriptBytes) + throw new TransactionEvidenceError('limit') + if (current.merklePath != null) continue + if (current.inputs.length === 0) throw new TransactionEvidenceError('invalid-evidence') + for (const input of current.inputs) { + if (input.sourceTransaction == null) throw new TransactionEvidenceError('invalid-evidence') + const sourceTxid = input.sourceTransaction.id('hex') + if (input.sourceTXID !== undefined && input.sourceTXID !== sourceTxid) + throw new TransactionEvidenceError('invalid-evidence') + const outpoint = `${sourceTxid}.${input.sourceOutputIndex}` + if (spent.has(outpoint)) throw new TransactionEvidenceError('invalid-evidence') + spent.add(outpoint) + pending.push(input.sourceTransaction) + } + } + return { + tx, + txid, + receipt: toHex(sha256(bytes)), + byteLength: bytes.length, + outputIndex, + graphBinding: toHex(sha256(tx.toBEEF())) + } +} + +/** Fence all owned transaction/proof bytes, including backend readiness callbacks. */ +export function assertEvidenceUnchanged(candidate: EvidenceCandidate): void { + if ( + candidate.tx.id('hex') !== candidate.txid || + toHex(sha256(candidate.tx.toBEEF())) !== candidate.graphBinding + ) { + throw new TransactionEvidenceError('invalid-evidence') + } +} diff --git a/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts b/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts new file mode 100644 index 000000000..921c67158 --- /dev/null +++ b/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts @@ -0,0 +1,583 @@ +import type ChainTracker from './ChainTracker.js' +import type BdkVerifierInterface from './BdkVerifierInterface.js' +import { scriptVerificationBackend } from './ScriptVerificationBackend.js' +import LockingScript from '../script/LockingScript.js' +import { EvidenceScriptWork, withEvidenceScriptWork } from './EvidenceScriptWork.js' +import { + defaultTransactionEvidenceLimits, + evidenceError, + assertEvidenceUnchanged, + parseEvidence, + TransactionEvidenceError, + type EvidenceCandidate, + type TransactionEvidence, + type TransactionEvidenceLimits, + type VerifiedTransactionOutput +} from './TransactionEvidence.js' + +/** Caller-controlled trust configuration; never populate this from lookup metadata. */ +export interface TransactionEvidenceContext { + chainTracker: ChainTracker + /** Network/genesis identifier or an explicit application chain namespace. */ + chainNamespace: string + /** Semantic verification policy/backend version; change it when policy changes. */ + policyId: string + verifier?: BdkVerifierInterface +} + +export interface TransactionEvidenceCoordinatorOptions extends TransactionEvidenceContext { + limits?: Partial +} + +interface Anchor { + root: string + height: number +} +interface Positive { + scripts: string[] + anchors: Anchor[] + observedHeight?: number + expiresAt: number + bytes: number +} +interface Consumer { + outputIndex: number + resolve: (value: VerifiedTransactionOutput) => void + reject: (error: TransactionEvidenceError) => void + detach: () => void +} +interface Work { + txid: string + revision: number + controller: AbortController + candidates: EvidenceCandidate[] + receipts: Set + consumers: Set + timer: ReturnType + expiresAt: number + running: boolean +} +interface ChainCall { + controller: AbortController + owners: number + promise: Promise +} + +function outcome(error: unknown): TransactionEvidenceError { + return evidenceError(error) +} + +/** + * Bounded, process-local transaction evidence work sharing. This is independent of + * lookup services, certificates and trust ratings. Positive reuse always checks + * canonical anchors again; ChainTracker remains the caller's trusted chain source. + * Synchronous parsing/script execution is byte/memory bounded, not preemptible. + */ +export class TransactionEvidenceCoordinator { + private readonly scriptWork: EvidenceScriptWork + readonly limits: Readonly + private context: TransactionEvidenceContext + private marker: string | number | undefined + private backend: BdkVerifierInterface | undefined + private revision = 0 + private disposed = false + private readonly work = new Map() + private readonly positives = new Map() + private readonly chainCalls = new Map() + private pendingChainCalls = 0 + private activeAttempts = 0 + private consumers = 0 + private retainedBytes = 0 + private expiryTimer?: ReturnType + + constructor(options: TransactionEvidenceCoordinatorOptions) { + this.context = this.validateContext(options) + this.limits = Object.freeze({ ...defaultTransactionEvidenceLimits, ...options.limits }) + for (const value of Object.values(this.limits)) { + if (!Number.isSafeInteger(value) || value <= 0) throw new TransactionEvidenceError('limit') + } + this.scriptWork = new EvidenceScriptWork(this.limits) + this.marker = this.context.chainTracker.getVerificationContext?.() + this.backend = this.context.verifier ?? scriptVerificationBackend() + } + + private validateContext(context: TransactionEvidenceContext): TransactionEvidenceContext { + if ( + context.chainTracker == null || + typeof context.chainTracker.isValidRootForHeight !== 'function' || + typeof context.chainTracker.currentHeight !== 'function' || + context.chainNamespace.length === 0 || + context.policyId.length === 0 + ) { + throw new TransactionEvidenceError('invalid-evidence') + } + return { + chainTracker: context.chainTracker, + chainNamespace: context.chainNamespace, + policyId: context.policyId, + verifier: context.verifier + } + } + + /** Explicit session/network/policy change. Stale in-flight work cannot publish. */ + setContext(context: TransactionEvidenceContext): void { + const next = this.validateContext(context) + this.invalidate(new TransactionEvidenceError('context-changed')) + this.context = next + this.marker = next.chainTracker.getVerificationContext?.() + this.backend = next.verifier ?? scriptVerificationBackend() + } + + dispose(): void { + this.disposed = true + this.invalidate(new TransactionEvidenceError('disposed')) + } + + private invalidate(error: TransactionEvidenceError): void { + this.revision++ + this.scriptWork.clear() + for (const job of this.work.values()) this.finish(job, undefined, error) + for (const txid of this.positives.keys()) this.removePositive(txid) + clearTimeout(this.expiryTimer) + this.expiryTimer = undefined + } + + private synchronize(): void { + if (this.disposed) throw new TransactionEvidenceError('disposed') + const marker = this.context.chainTracker.getVerificationContext?.() + const backend = this.context.verifier ?? scriptVerificationBackend() + if (marker !== this.marker || backend !== this.backend) { + this.invalidate(new TransactionEvidenceError('context-changed')) + this.marker = marker + this.backend = backend + } + } + + private check(job: Work, signal: AbortSignal): void { + this.synchronize() + if (job.revision !== this.revision) throw new TransactionEvidenceError('context-changed') + if (Date.now() >= job.expiresAt) throw new TransactionEvidenceError('timeout') + if (signal.aborted || job.consumers.size === 0) + throw outcome(signal.reason ?? new TransactionEvidenceError('cancelled')) + } + + /** Snapshot intake is synchronous up to the returned Promise's first await. */ + async verify( + evidence: TransactionEvidence, + options: { signal?: AbortSignal } = {} + ): Promise { + this.synchronize() + this.prune() + if (options.signal?.aborted === true) throw new TransactionEvidenceError('cancelled') + if (this.consumers >= this.limits.consumers) throw new TransactionEvidenceError('limit') + let candidate: EvidenceCandidate + try { + candidate = parseEvidence(evidence, this.limits) + } catch (error) { + throw outcome(error) + } + let job = this.work.get(candidate.txid) + if (job === undefined) { + if (this.work.size >= this.limits.pendingTransactions) + throw new TransactionEvidenceError('limit') + const controller = new AbortController() + const created: Work = { + txid: candidate.txid, + revision: this.revision, + controller, + candidates: [], + receipts: new Set(), + consumers: new Set(), + running: false, + timer: setTimeout( + () => this.finish(created, undefined, new TransactionEvidenceError('timeout')), + this.limits.requestTimeoutMs + ), + expiresAt: Date.now() + this.limits.requestTimeoutMs + } + job = created + this.work.set(job.txid, job) + } + if (!job.receipts.has(candidate.receipt)) { + if ( + job.receipts.size >= this.limits.candidatesPerTransaction || + this.retainedBytes + candidate.byteLength > this.limits.retainedBytes + ) { + if (job.consumers.size === 0) + this.finish(job, undefined, new TransactionEvidenceError('limit')) + throw new TransactionEvidenceError('limit') + } + job.receipts.add(candidate.receipt) + job.candidates.push(candidate) + this.retainedBytes += candidate.byteLength + } + const current = job + return await new Promise((resolve, reject) => { + const cancel = (): void => { + this.removeConsumer(current, consumer) + reject(new TransactionEvidenceError('cancelled')) + if (current.consumers.size === 0) + this.finish(current, undefined, new TransactionEvidenceError('cancelled')) + } + const consumer: Consumer = { + outputIndex: candidate.outputIndex, + resolve, + reject, + detach: () => options.signal?.removeEventListener('abort', cancel) + } + current.consumers.add(consumer) + this.consumers++ + options.signal?.addEventListener('abort', cancel, { once: true }) + this.pump() + }) + } + + private removeConsumer(job: Work, consumer: Consumer): void { + if (!job.consumers.delete(consumer)) return + consumer.detach() + this.consumers-- + } + + private finish( + job: Work, + positive?: Positive, + error = new TransactionEvidenceError('invalid-evidence') + ): void { + if (this.work.get(job.txid) !== job) return + this.work.delete(job.txid) + clearTimeout(job.timer) + for (const candidate of job.candidates) this.retainedBytes -= candidate.byteLength + job.candidates = [] + for (const consumer of job.consumers) { + this.removeConsumer(job, consumer) + if (positive === undefined) consumer.reject(error) + else + consumer.resolve({ + txid: job.txid, + outputIndex: consumer.outputIndex, + outpoint: `${job.txid}.${consumer.outputIndex}`, + lockingScript: LockingScript.fromHex(positive.scripts[consumer.outputIndex]) + }) + } + job.controller.abort(error) + } + + private pump(): void { + let running = [...this.work.values()].filter(job => job.running).length + for (const job of this.work.values()) { + if ( + running >= this.limits.concurrentTransactions || + this.activeAttempts >= this.limits.concurrentTransactions + ) + break + if (job.running) continue + job.running = true + running++ + void this.run(job).finally(() => this.pump()) + } + } + + private async run(job: Work): Promise { + let error = new TransactionEvidenceError('invalid-evidence') + try { + const cached = this.positives.get(job.txid) + if (cached !== undefined) { + try { + await this.attempt(job, async signal => await this.recheck(job, cached, signal)) + this.check(job, job.controller.signal) + this.finish(job, cached) + return + } catch (failure) { + error = outcome(failure) + this.removePositive(job.txid) + } + } + while (job.candidates.length > 0 && !job.controller.signal.aborted) { + const candidate = job.candidates.shift()! + try { + const positive = await this.attempt( + job, + async signal => { + const token = await this.contextToken(job, signal) + const anchors = new Map() + let observedHeight: number | undefined + const tracker: ChainTracker = { + isValidRootForHeight: async (root, height) => { + const valid = await this.chainCall( + job, + `root:${height}:${root}`, + signal, + root, + height + ) + if (valid === true) anchors.set(`${height}:${root}`, { root, height }) + return valid === true + }, + currentHeight: async () => { + const height = await this.chainCall(job, 'height', signal) + if (typeof height !== 'number' || !Number.isSafeInteger(height) || height < 0) + throw new TransactionEvidenceError('invalid-evidence') + observedHeight = Math.max(observedHeight ?? 0, height) + return height + } + } + this.check(job, signal) + const valid = await withEvidenceScriptWork( + candidate.tx, + { + work: this.scriptWork, + signal, + check: () => this.check(job, signal) + }, + async () => + await candidate.tx.verify( + tracker, + undefined, + this.limits.scriptMemoryBytes, + this.backend + ) + ) + this.check(job, signal) + assertEvidenceUnchanged(candidate) + if (valid !== true || anchors.size === 0) + throw new TransactionEvidenceError('invalid-evidence') + const scripts = candidate.tx.outputs.map(output => output.lockingScript.toHex()) + const positive: Positive = { + scripts, + anchors: [...anchors.values()], + observedHeight, + expiresAt: Date.now() + this.limits.cacheAgeMs, + bytes: + scripts.reduce((total, script) => total + script.length / 2, 0) + + anchors.size * 40 + } + await this.recheck(job, positive, signal, token) + assertEvidenceUnchanged(candidate) + return positive + }, + () => { + this.retainedBytes -= candidate.byteLength + } + ) + this.check(job, job.controller.signal) + this.cache(job.txid, positive) + this.finish(job, positive) + return + } catch (failure) { + error = outcome(failure) + } + } + } catch (failure) { + error = outcome(failure) + } + this.finish(job, undefined, error) + } + + private async attempt( + job: Work, + operation: (signal: AbortSignal) => Promise, + settled?: () => void + ): Promise { + if (this.activeAttempts >= this.limits.concurrentTransactions) { + settled?.() + throw new TransactionEvidenceError('limit') + } + this.activeAttempts++ + const controller = new AbortController() + const deadline = Date.now() + this.limits.attemptTimeoutMs + const abort = (): void => controller.abort(job.controller.signal.reason) + job.controller.signal.addEventListener('abort', abort, { once: true }) + const timer = setTimeout( + () => controller.abort(new TransactionEvidenceError('timeout')), + this.limits.attemptTimeoutMs + ) + let rejectAbort: () => void = () => {} + const cancelled = new Promise((_resolve, reject) => { + rejectAbort = () => reject(outcome(controller.signal.reason)) + controller.signal.addEventListener('abort', rejectAbort, { once: true }) + }) + // Keep non-abortable backend work counted until its actual Promise settles. + const pending = operation(controller.signal) + .then(value => { + if (Date.now() >= deadline) throw new TransactionEvidenceError('timeout') + return value + }) + .finally(() => { + settled?.() + this.activeAttempts-- + this.pump() + }) + try { + return await Promise.race([pending, cancelled]) + } finally { + clearTimeout(timer) + job.controller.signal.removeEventListener('abort', abort) + controller.signal.removeEventListener('abort', rejectAbort) + } + } + + private async chainCall( + job: Work, + key: string, + signal: AbortSignal, + root?: string, + height?: number + ): Promise { + this.check(job, signal) + const scopedKey = `${job.revision}:${key}` + let call = this.chainCalls.get(scopedKey) + if (call === undefined) { + if (this.pendingChainCalls >= this.limits.pendingChainCalls) + throw new TransactionEvidenceError('limit') + const controller = new AbortController() + const tracker = this.context.chainTracker + const created: ChainCall = { controller, owners: 0, promise: Promise.resolve(false) } + this.pendingChainCalls++ + created.promise = Promise.resolve() + .then(async () => { + if (controller.signal.aborted) throw new TransactionEvidenceError('cancelled') + if (key === 'context-token') + return await tracker.getVerificationContextToken!(controller.signal) + return root === undefined + ? await tracker.currentHeight(controller.signal) + : await tracker.isValidRootForHeight(root, height!, controller.signal) + }) + .finally(() => { + this.pendingChainCalls-- + if (this.chainCalls.get(scopedKey) === created) this.chainCalls.delete(scopedKey) + }) + call = created + this.chainCalls.set(scopedKey, created) + } + call.owners++ + const owned = call + let abort: () => void = () => {} + const cancelled = new Promise((_resolve, reject) => { + abort = () => reject(outcome(signal.reason)) + signal.addEventListener('abort', abort, { once: true }) + }) + try { + const value = await Promise.race([owned.promise, cancelled]) + this.check(job, signal) + if (root !== undefined && value !== true) this.invalidateAnchor(root, height!) + return value + } finally { + signal.removeEventListener('abort', abort) + owned.owners-- + if (owned.owners === 0) { + if (this.chainCalls.get(scopedKey) === owned) this.chainCalls.delete(scopedKey) + owned.controller.abort() + } + } + } + + private async contextToken(job: Work, signal: AbortSignal): Promise { + if (this.context.chainTracker.getVerificationContextToken === undefined) return undefined + const token = await this.chainCall(job, 'context-token', signal) + if (typeof token !== 'string' || token.length === 0) + throw new TransactionEvidenceError('invalid-evidence') + if (token.length > 4096) throw new TransactionEvidenceError('limit') + return token + } + + private async recheck( + job: Work, + positive: Positive, + signal: AbortSignal, + token?: string + ): Promise { + const before = token ?? (await this.contextToken(job, signal)) + for (const anchor of positive.anchors) { + if ( + (await this.chainCall( + job, + `root:${anchor.height}:${anchor.root}`, + signal, + anchor.root, + anchor.height + )) !== true + ) { + throw new TransactionEvidenceError('invalid-evidence') + } + } + if (positive.observedHeight !== undefined) { + const height = await this.chainCall(job, 'height', signal) + if (typeof height !== 'number' || height < positive.observedHeight) + throw new TransactionEvidenceError('invalid-evidence') + } + if ((await this.contextToken(job, signal)) !== before) + throw new TransactionEvidenceError('context-changed') + this.check(job, signal) + } + + private invalidateAnchor(root: string, height: number): void { + this.scriptWork.clear() + for (const [txid, positive] of this.positives) { + if (positive.anchors.some(anchor => anchor.root === root && anchor.height === height)) + this.removePositive(txid) + } + } + + private removePositive(txid: string): void { + const positive = this.positives.get(txid) + if (positive === undefined) return + this.retainedBytes -= positive.bytes + this.positives.delete(txid) + } + + private cache(txid: string, positive: Positive): void { + this.removePositive(txid) + this.prune() + while ( + this.positives.size > 0 && + (this.positives.size >= this.limits.cacheEntries || + this.retainedBytes + positive.bytes > this.limits.retainedBytes) + ) { + this.removePositive(this.positives.keys().next().value!) + } + if (this.retainedBytes + positive.bytes <= this.limits.retainedBytes) { + this.positives.set(txid, positive) + this.retainedBytes += positive.bytes + this.scheduleExpiry() + } + } + + private prune(): void { + for (const [txid, positive] of this.positives) + if (positive.expiresAt <= Date.now()) this.removePositive(txid) + } + + private scheduleExpiry(): void { + clearTimeout(this.expiryTimer) + const expiresAt = Math.min(...[...this.positives.values()].map(value => value.expiresAt)) + if (!Number.isFinite(expiresAt)) return + this.expiryTimer = setTimeout( + () => { + this.prune() + this.scheduleExpiry() + }, + Math.max(1, expiresAt - Date.now()) + ) + this.expiryTimer.unref?.() + } + + /** Payload-free local diagnostics; pending calls include abandoned, non-abortable I/O. */ + getStats(): { + pendingTransactions: number + consumers: number + cachedTransactions: number + retainedBytes: number + pendingChainCalls: number + activeAttempts: number + } { + this.prune() + return { + pendingTransactions: this.work.size, + consumers: this.consumers, + cachedTransactions: this.positives.size, + retainedBytes: this.retainedBytes, + pendingChainCalls: this.pendingChainCalls, + activeAttempts: this.activeAttempts + } + } +} diff --git a/packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts b/packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts new file mode 100644 index 000000000..456956973 --- /dev/null +++ b/packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts @@ -0,0 +1,1085 @@ +import ChainTracker from '../ChainTracker' +import MerklePath from '../MerklePath' +import Transaction from '../Transaction' +import { TransactionEvidenceError } from '../TransactionEvidence' +import { TransactionEvidenceCoordinator } from '../TransactionEvidenceCoordinator' +import type BdkVerifierInterface from '../BdkVerifierInterface' +import type { BdkVerifyScriptsParams } from '../BdkVerifierInterface' +import P2PKH from '../../script/templates/P2PKH' +import PrivateKey from '../../primitives/PrivateKey' +import Script from '../../script/Script' +import Spend from '../../script/Spend' +import ScriptResourceLimitError from '../../script/ScriptResourceLimitError' + +const height = 700_000 + +interface Deferred { + promise: Promise + resolve: (value: T) => void + reject: (error: unknown) => void +} + +function deferred(): Deferred { + let resolve!: (value: T) => void + let reject!: (error: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +class LocalChainTracker implements ChainTracker { + readonly roots = new Set() + readonly calls: Array<{ root: string; height: number }> = [] + context = 0 + current = height + 101 + gate: Promise | undefined + onRootCall: (() => void) | undefined + aborts = 0 + + async currentHeight(): Promise { + return this.current + } + + async isValidRootForHeight( + root: string, + blockHeight: number, + signal?: AbortSignal + ): Promise { + this.calls.push({ root, height: blockHeight }) + this.onRootCall?.() + if (signal != null) + signal.addEventListener( + 'abort', + () => { + this.aborts++ + }, + { once: true } + ) + if (this.gate !== undefined) await this.gate + if (signal?.aborted === true) throw new TransactionEvidenceError('cancelled') + return blockHeight === height && this.roots.has(root) + } + + getVerificationContext(): number { + return this.context + } +} + +class TokenChainTracker extends LocalChainTracker { + readonly rootsByHeight = new Map>() + readonly rootChecks = new Map() + token = 'tip-a' + tokenGate: Promise | undefined + tokenCalls = 0 + tokenAborts = 0 + tokenFailure: Error | undefined + onAnchorCheck: ((height: number, count: number) => void) | undefined + + addRoot(root: string, blockHeight: number): void { + const roots = this.rootsByHeight.get(blockHeight) ?? new Set() + roots.add(root) + this.rootsByHeight.set(blockHeight, roots) + } + + removeRoot(root: string, blockHeight: number): void { + this.rootsByHeight.get(blockHeight)?.delete(root) + } + + async isValidRootForHeight( + root: string, + blockHeight: number, + signal?: AbortSignal + ): Promise { + this.calls.push({ root, height: blockHeight }) + this.onRootCall?.() + if (signal != null) + signal.addEventListener( + 'abort', + () => { + this.aborts++ + }, + { once: true } + ) + if (this.gate !== undefined) await this.gate + if (signal?.aborted === true) throw new TransactionEvidenceError('cancelled') + const count = (this.rootChecks.get(blockHeight) ?? 0) + 1 + this.rootChecks.set(blockHeight, count) + const valid = this.rootsByHeight.get(blockHeight)?.has(root) === true + this.onAnchorCheck?.(blockHeight, count) + return valid + } + + async getVerificationContextToken(signal?: AbortSignal): Promise { + this.tokenCalls++ + if (signal != null) + signal.addEventListener( + 'abort', + () => { + this.tokenAborts++ + }, + { once: true } + ) + if (this.tokenGate !== undefined) await this.tokenGate + if (signal?.aborted === true) throw new TransactionEvidenceError('cancelled') + if (this.tokenFailure !== undefined) throw this.tokenFailure + return this.token + } +} + +async function fixture(): Promise<{ + tracker: LocalChainTracker + tx: Transaction + evidence: number[] +}> { + const key = new PrivateKey(42) + const p2pkh = new P2PKH() + const tracker = new LocalChainTracker() + const source = new Transaction() + source.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + source.addOutput({ satoshis: 10, lockingScript: p2pkh.lock(key.toAddress()) }) + source.merklePath = new MerklePath(height, [ + [ + { offset: 0, hash: source.id('hex'), txid: true }, + { offset: 1, duplicate: true } + ] + ]) + tracker.roots.add(source.merklePath.computeRoot(source.id('hex'))) + + const tx = new Transaction() + tx.addInput({ + sourceTransaction: source, + sourceOutputIndex: 0, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + tx.addOutput({ satoshis: 4, lockingScript: p2pkh.lock(key.toAddress()) }) + tx.addOutput({ satoshis: 4, lockingScript: p2pkh.lock(key.toAddress()) }) + await tx.sign() + return { tracker, tx, evidence: tx.toBEEF() } +} + +async function sharedAncestorFixture(): Promise<{ + tracker: LocalChainTracker + ancestor: Transaction + first: Transaction + second: Transaction + firstEvidence: number[] + secondEvidence: number[] +}> { + const key = new PrivateKey(43) + const p2pkh = new P2PKH() + const tracker = new LocalChainTracker() + const confirmed = new Transaction() + confirmed.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + confirmed.addOutput({ satoshis: 20, lockingScript: p2pkh.lock(key.toAddress()) }) + confirmed.merklePath = new MerklePath(height, [ + [ + { offset: 0, hash: confirmed.id('hex'), txid: true }, + { offset: 1, duplicate: true } + ] + ]) + tracker.roots.add(confirmed.merklePath.computeRoot(confirmed.id('hex'))) + + const ancestor = new Transaction() + ancestor.addInput({ + sourceTransaction: confirmed, + sourceOutputIndex: 0, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + ancestor.addOutput({ satoshis: 9, lockingScript: p2pkh.lock(key.toAddress()) }) + ancestor.addOutput({ satoshis: 9, lockingScript: p2pkh.lock(key.toAddress()) }) + await ancestor.sign() + + const createChild = async (outputIndex: number): Promise => { + const child = new Transaction() + child.addInput({ + sourceTransaction: ancestor, + sourceOutputIndex: outputIndex, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + child.addOutput({ satoshis: 8, lockingScript: p2pkh.lock(key.toAddress()) }) + await child.sign() + return child + } + const first = await createChild(0) + const second = await createChild(1) + return { + tracker, + ancestor, + first, + second, + firstEvidence: first.toBEEF(), + secondEvidence: second.toBEEF() + } +} + +async function twoAnchorFixture(): Promise<{ + tracker: TokenChainTracker + tx: Transaction + evidence: number[] + roots: string[] +}> { + const key = new PrivateKey(44) + const p2pkh = new P2PKH() + const tracker = new TokenChainTracker() + const roots: string[] = [] + const sources: Transaction[] = [] + for (const blockHeight of [200, 100]) { + const source = new Transaction() + source.addInput({ + sourceTXID: blockHeight.toString(16).padStart(64, '0'), + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + source.addOutput({ satoshis: 10, lockingScript: p2pkh.lock(key.toAddress()) }) + source.merklePath = new MerklePath(blockHeight, [ + [ + { offset: 0, hash: source.id('hex'), txid: true }, + { offset: 1, duplicate: true } + ] + ]) + const root = source.merklePath.computeRoot(source.id('hex')) + tracker.addRoot(root, blockHeight) + roots.push(root) + sources.push(source) + } + const tx = new Transaction() + for (const source of sources) { + tx.addInput({ + sourceTransaction: source, + sourceOutputIndex: 0, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + } + tx.addOutput({ satoshis: 18, lockingScript: p2pkh.lock(key.toAddress()) }) + await tx.sign() + return { tracker, tx, evidence: tx.toBEEF(), roots } +} + +function verifyParamsInJavaScript(params: BdkVerifyScriptsParams): boolean { + const sigHashCache = { hashOutputsSingle: new Map() } + for (const [inputIndex, input] of params.tx.inputs.entries()) { + const source = input.sourceTransaction + const sourceOutput = source?.outputs[input.sourceOutputIndex] + if (source === undefined || sourceOutput === undefined || input.unlockingScript === undefined) + return false + const valid = new Spend({ + sourceTXID: input.sourceTXID ?? source.id('hex'), + sourceOutputIndex: input.sourceOutputIndex, + lockingScript: sourceOutput.lockingScript, + sourceSatoshis: sourceOutput.satoshis ?? 0, + transactionVersion: params.tx.version, + otherInputs: [], + allInputs: params.tx.inputs, + unlockingScript: input.unlockingScript, + inputSequence: input.sequence ?? 0xffffffff, + inputIndex, + outputs: params.tx.outputs, + lockTime: params.tx.lockTime, + memoryLimit: params.memoryLimit, + sigHashCache + }).validateJavaScript() + if (!valid) return false + } + return true +} + +function alternateReceipt(evidence: number[], marker: number): number[] { + const alternate = Transaction.fromBEEF(evidence) + const source = alternate.inputs[0].sourceTransaction + if (source === undefined) throw new Error('fixture source is missing') + source.merklePath = new MerklePath(height, [ + [ + { offset: 0, hash: source.id('hex'), txid: true }, + { offset: 1, hash: marker.toString(16).padStart(64, '0') } + ] + ]) + return alternate.toBEEF() +} + +function coordinator( + tracker: LocalChainTracker, + limits = {}, + verifier?: BdkVerifierInterface +): TransactionEvidenceCoordinator { + return new TransactionEvidenceCoordinator({ + chainTracker: tracker, + chainNamespace: 'local-canonical-chain', + policyId: 'p2pkh-consensus', + limits, + verifier + }) +} + +async function expectCode( + promise: Promise, + code: TransactionEvidenceError['code'] +): Promise { + await expect(promise).rejects.toMatchObject({ code }) +} + +describe('TransactionEvidenceCoordinator', () => { + it('coalesces concurrent output requests for one signed transaction into one full verification', async () => { + const { tracker, tx, evidence } = await fixture() + const verify = jest.spyOn(Transaction.prototype, 'verify') + const verifier: BdkVerifierInterface = { + supportsMemoryLimit: true, + verifyScripts: jest.fn(async () => true) + } + const subject = coordinator(tracker, {}, verifier) + + const [first, second] = await Promise.all([ + subject.verify({ beef: evidence, outputIndex: 0, txid: tx.id('hex') }), + subject.verify({ beef: evidence, outputIndex: 1, txid: tx.id('hex') }) + ]) + + expect(first).toMatchObject({ + txid: tx.id('hex'), + outputIndex: 0, + outpoint: `${tx.id('hex')}.0` + }) + expect(second).toMatchObject({ + txid: tx.id('hex'), + outputIndex: 1, + outpoint: `${tx.id('hex')}.1` + }) + expect(first.lockingScript.toHex()).toBe(tx.outputs[0].lockingScript.toHex()) + expect(tracker.calls).toContainEqual({ root: [...tracker.roots][0], height }) + expect(verify).toHaveBeenCalledTimes(1) + expect(verifier.verifyScripts).toHaveBeenCalledTimes(1) + expect(subject.getStats()).toMatchObject({ + pendingTransactions: 0, + consumers: 0, + activeAttempts: 0 + }) + verify.mockRestore() + }) + + it('does not permanently poison a transaction key after an invalid canonical root', async () => { + const { tracker, tx, evidence } = await fixture() + const root = [...tracker.roots][0] + tracker.roots.clear() + const subject = coordinator(tracker) + + await expectCode(subject.verify({ beef: evidence, outputIndex: 0 }), 'invalid-evidence') + tracker.roots.add(root) + await expect(subject.verify({ beef: evidence, outputIndex: 0 })).resolves.toMatchObject({ + txid: tx.id('hex') + }) + }) + + it('retains a distinct same-target receipt when a later proof repairs an earlier failed root', async () => { + const { tracker, evidence } = await fixture() + const alternate = Transaction.fromBEEF(evidence) + const source = alternate.inputs[0].sourceTransaction + if (source === undefined) throw new Error('fixture source is missing') + source.merklePath = new MerklePath(height, [ + [ + { offset: 0, hash: source.id('hex'), txid: true }, + { offset: 1, hash: '42'.repeat(32) } + ] + ]) + tracker.roots.clear() + tracker.roots.add(source.merklePath.computeRoot(source.id('hex'))) + const subject = coordinator(tracker) + + const [fromBadReceipt, fromGoodReceipt] = await Promise.all([ + subject.verify({ beef: evidence, outputIndex: 0 }), + subject.verify({ beef: alternate.toBEEF(), outputIndex: 1 }) + ]) + + expect(fromBadReceipt).toMatchObject({ outputIndex: 0 }) + expect(fromGoodReceipt).toMatchObject({ outputIndex: 1 }) + }) + + it('lets one consumer cancel while a coalesced consumer still receives the verified output', async () => { + const { tracker, evidence } = await fixture() + const release = deferred() + tracker.gate = release.promise + const subject = coordinator(tracker) + const abort = new AbortController() + const entered = deferred() + tracker.onRootCall = () => entered.resolve() + const cancelled = subject.verify({ beef: evidence, outputIndex: 0 }, { signal: abort.signal }) + const successful = subject.verify({ beef: evidence, outputIndex: 1 }) + + await entered.promise + abort.abort() + await expectCode(cancelled, 'cancelled') + // The shared chain call still has a live owner, so this cancellation cannot abort it. + expect(tracker.aborts).toBe(0) + release.resolve() + await expect(successful).resolves.toMatchObject({ outputIndex: 1 }) + }) + + it('aborts the last owner, leaves no late cache publication, and counts non-abortable work until it settles', async () => { + const { tracker, evidence } = await fixture() + const release = deferred() + tracker.gate = release.promise + const subject = coordinator(tracker) + const abort = new AbortController() + const entered = deferred() + tracker.onRootCall = () => entered.resolve() + const request = subject.verify({ beef: evidence, outputIndex: 0 }, { signal: abort.signal }) + + await entered.promise + abort.abort() + await expectCode(request, 'cancelled') + expect(tracker.aborts).toBe(1) + expect(subject.getStats().pendingChainCalls).toBe(1) + release.resolve() + await new Promise(resolve => setImmediate(resolve)) + expect(subject.getStats()).toMatchObject({ + cachedTransactions: 0, + pendingChainCalls: 0, + activeAttempts: 0 + }) + }) + + it('fences in-flight and cached work when namespace or policy context changes', async () => { + const { tracker, evidence } = await fixture() + const release = deferred() + tracker.gate = release.promise + const subject = coordinator(tracker) + const previous = subject.verify({ beef: evidence, outputIndex: 0 }) + + subject.setContext({ + chainTracker: tracker, + chainNamespace: 'replacement-chain', + policyId: 'replacement-policy' + }) + await expectCode(previous, 'context-changed') + release.resolve() + await expect(subject.verify({ beef: evidence, outputIndex: 0 })).resolves.toMatchObject({ + outputIndex: 0 + }) + }) + + it('rechecks cached canonical dependencies on a reorganization and retains a valid cache at a new tip', async () => { + const { tracker, evidence } = await fixture() + const root = [...tracker.roots][0] + const verify = jest.spyOn(Transaction.prototype, 'verify') + const verifier: BdkVerifierInterface = { + supportsMemoryLimit: true, + verifyScripts: jest.fn(async () => true) + } + const subject = coordinator(tracker, {}, verifier) + + await subject.verify({ beef: evidence, outputIndex: 0 }) + expect(verify).toHaveBeenCalledTimes(1) + expect(verifier.verifyScripts).toHaveBeenCalledTimes(1) + tracker.current++ + await expect(subject.verify({ beef: evidence, outputIndex: 0 })).resolves.toMatchObject({ + outputIndex: 0 + }) + expect(verify).toHaveBeenCalledTimes(1) + expect(verifier.verifyScripts).toHaveBeenCalledTimes(1) + tracker.roots.clear() + await expectCode(subject.verify({ beef: evidence, outputIndex: 0 }), 'invalid-evidence') + expect(verify).toHaveBeenCalledTimes(2) + // The rejected canonical dependency stops before target script execution. + expect(verifier.verifyScripts).toHaveBeenCalledTimes(1) + expect(tracker.calls).toContainEqual({ root, height }) + verify.mockRestore() + }) + + it('times out an attempt without poisoning a later valid receipt', async () => { + jest.useFakeTimers() + try { + const { tracker, evidence } = await fixture() + const release = deferred() + const entered = deferred() + tracker.gate = release.promise + tracker.onRootCall = () => entered.resolve() + const subject = coordinator(tracker, { attemptTimeoutMs: 10, requestTimeoutMs: 100 }) + const timedOut = subject.verify({ beef: evidence, outputIndex: 0 }) + + await entered.promise + await jest.advanceTimersByTimeAsync(11) + await expectCode(timedOut, 'timeout') + release.resolve() + await jest.runOnlyPendingTimersAsync() + tracker.gate = undefined + await expect(subject.verify({ beef: evidence, outputIndex: 0 })).resolves.toMatchObject({ + outputIndex: 0 + }) + } finally { + jest.useRealTimers() + } + }) + + it('drops an in-flight result when the tracker changes its verification context', async () => { + const { tracker, evidence } = await fixture() + const release = deferred() + tracker.gate = release.promise + const subject = coordinator(tracker) + const request = subject.verify({ beef: evidence, outputIndex: 0 }) + + tracker.context++ + release.resolve() + await expectCode(request, 'context-changed') + expect(subject.getStats().cachedTransactions).toBe(0) + }) + + it('snapshots bytes before deriving its key and rejects a mismatched txid hint', async () => { + const { tracker, tx, evidence } = await fixture() + const subject = coordinator(tracker) + const mutable = evidence.slice() + const request = subject.verify({ beef: mutable, outputIndex: 0, txid: tx.id('hex') }) + mutable.fill(0) + await expect(request).resolves.toMatchObject({ txid: tx.id('hex') }) + await expectCode( + subject.verify({ beef: evidence, outputIndex: 0, txid: '00'.repeat(32) }), + 'invalid-evidence' + ) + }) + + it('enforces finite admission, consumer, and cache-expiry limits', async () => { + const { tracker, evidence } = await fixture() + const tooSmall = coordinator(tracker, { candidateBytes: 1 }) + await expectCode(tooSmall.verify({ beef: evidence, outputIndex: 0 }), 'limit') + + const gate = deferred() + tracker.gate = gate.promise + const oneConsumer = coordinator(tracker, { consumers: 1 }) + const first = oneConsumer.verify({ beef: evidence, outputIndex: 0 }) + await expectCode(oneConsumer.verify({ beef: evidence, outputIndex: 1 }), 'limit') + gate.resolve() + await expect(first).resolves.toMatchObject({ outputIndex: 0 }) + + jest.useFakeTimers() + try { + const verify = jest.spyOn(Transaction.prototype, 'verify') + const expiring = coordinator(tracker, { cacheAgeMs: 1 }) + await expiring.verify({ beef: evidence, outputIndex: 0 }) + await jest.advanceTimersByTimeAsync(2) + await expiring.verify({ beef: evidence, outputIndex: 0 }) + expect(verify).toHaveBeenCalledTimes(2) + verify.mockRestore() + } finally { + jest.useRealTimers() + } + }) + + it('keeps a successful shared ancestor when a sibling non-batch script execution rejects', async () => { + const { tracker, ancestor, first, second, firstEvidence, secondEvidence } = + await sharedAncestorFixture() + const entered = deferred() + const release = deferred() + const verifier: BdkVerifierInterface = { + supportsMemoryLimit: true, + verifyScripts: async params => { + if (params.tx.id('hex') === ancestor.id('hex')) { + entered.resolve() + await release.promise + return true + } + if (params.tx.id('hex') === first.id('hex')) { + throw new ScriptResourceLimitError('stack', 1, 2) + } + return verifyParamsInJavaScript(params) + } + } + const subject = coordinator(tracker, {}, verifier) + const failed = subject.verify({ beef: firstEvidence, outputIndex: 0 }) + await entered.promise + const succeeded = subject.verify({ beef: secondEvidence, outputIndex: 0 }) + release.resolve() + + await expectCode(failed, 'limit') + await expect(succeeded).resolves.toMatchObject({ + txid: second.id('hex'), + outputIndex: 0 + }) + }) + + it('rejects every created batch entry when the backend batch call itself fails', async () => { + const { tracker, firstEvidence, secondEvidence } = await sharedAncestorFixture() + const entered = deferred() + const release = deferred() + const verifier: BdkVerifierInterface = { + supportsMemoryLimit: true, + verifyScripts: async () => true, + verifyScriptsBatch: async () => { + entered.resolve() + await release.promise + throw new ScriptResourceLimitError('stack', 1, 2) + } + } + const subject = coordinator(tracker, {}, verifier) + const first = subject.verify({ beef: firstEvidence, outputIndex: 0 }) + await entered.promise + const second = subject.verify({ beef: secondEvidence, outputIndex: 0 }) + release.resolve() + + await expectCode(first, 'limit') + await expectCode(second, 'limit') + }) + + it('runs a signed shared ancestor once while each distinct child still validates its own input', async () => { + const { tracker, first, second, firstEvidence, secondEvidence } = await sharedAncestorFixture() + const validate = jest.spyOn(Spend.prototype, 'validateJavaScript') + const subject = coordinator(tracker) + + await Promise.all([ + subject.verify({ beef: firstEvidence, outputIndex: 0 }), + subject.verify({ beef: secondEvidence, outputIndex: 0 }) + ]) + + expect(first.id('hex')).not.toBe(second.id('hex')) + expect(validate).toHaveBeenCalledTimes(3) + validate.mockRestore() + }) + + it('preserves backend batches while sharing the in-flight signed ancestor exactly once', async () => { + const { tracker, ancestor, first, second, firstEvidence, secondEvidence } = + await sharedAncestorFixture() + const validate = jest.spyOn(Spend.prototype, 'validateJavaScript') + const verifyScripts = jest.fn(async () => { + throw new Error('batch path required') + }) + const verifyScriptsBatch = jest.fn(async (params: readonly BdkVerifyScriptsParams[]) => + params.map(verifyParamsInJavaScript) + ) + const verifier: BdkVerifierInterface = { + supportsMemoryLimit: true, + verifyScripts, + verifyScriptsBatch + } + const subject = coordinator(tracker, {}, verifier) + + await Promise.all([ + subject.verify({ beef: firstEvidence, outputIndex: 0 }), + subject.verify({ beef: secondEvidence, outputIndex: 0 }) + ]) + + const submitted = verifyScriptsBatch.mock.calls.flatMap(([params]) => + params.map(param => param.tx.id('hex')) + ) + expect(verifyScripts).not.toHaveBeenCalled() + expect(verifyScriptsBatch.mock.calls.some(([params]) => params.length > 1)).toBe(true) + expect(submitted.filter(txid => txid === ancestor.id('hex'))).toHaveLength(1) + expect(submitted).toEqual(expect.arrayContaining([first.id('hex'), second.id('hex')])) + expect(validate).toHaveBeenCalledTimes(3) + validate.mockRestore() + }) + + it('keeps a shared ancestor alive when one distinct-target owner cancels', async () => { + const { tracker, ancestor, firstEvidence, secondEvidence } = await sharedAncestorFixture() + const entered = deferred() + const release = deferred() + const batches: string[][] = [] + const verifier: BdkVerifierInterface = { + supportsMemoryLimit: true, + verifyScripts: async () => false, + verifyScriptsBatch: async params => { + batches.push(params.map(param => param.tx.id('hex'))) + entered.resolve() + await release.promise + return params.map(verifyParamsInJavaScript) + } + } + const subject = coordinator(tracker, {}, verifier) + const abort = new AbortController() + const cancelled = subject.verify( + { beef: firstEvidence, outputIndex: 0 }, + { signal: abort.signal } + ) + const successful = subject.verify({ beef: secondEvidence, outputIndex: 0 }) + + await entered.promise + abort.abort() + await expectCode(cancelled, 'cancelled') + release.resolve() + await expect(successful).resolves.toMatchObject({ outputIndex: 0 }) + expect(batches.flat().filter(txid => txid === ancestor.id('hex'))).toHaveLength(1) + }) + + it('does not publish a late backend completion across a policy context change', async () => { + const { tracker, firstEvidence } = await sharedAncestorFixture() + const entered = deferred() + const release = deferred() + const verifier: BdkVerifierInterface = { + supportsMemoryLimit: true, + verifyScripts: async () => false, + verifyScriptsBatch: async params => { + entered.resolve() + await release.promise + return params.map(verifyParamsInJavaScript) + } + } + const subject = coordinator(tracker, {}, verifier) + const request = subject.verify({ beef: firstEvidence, outputIndex: 0 }) + + await entered.promise + subject.setContext({ + chainTracker: tracker, + chainNamespace: 'local-canonical-chain', + policyId: 'replacement-policy', + verifier + }) + await expectCode(request, 'context-changed') + release.resolve() + await new Promise(resolve => setImmediate(resolve)) + expect(subject.getStats()).toMatchObject({ cachedTransactions: 0, activeAttempts: 0 }) + await expect(subject.verify({ beef: firstEvidence, outputIndex: 0 })).resolves.toMatchObject({ + outputIndex: 0 + }) + }) + + it('rejects backend mutation of an owned verification parameter and accepts a later clean receipt', async () => { + const { tracker, firstEvidence } = await sharedAncestorFixture() + const entered = deferred() + const release = deferred() + let mutate = true + const verifier: BdkVerifierInterface = { + supportsMemoryLimit: true, + verifyScripts: async () => false, + verifyScriptsBatch: async params => { + entered.resolve() + await release.promise + if (mutate) { + const source = params[0].tx.inputs[0].sourceTransaction + if (source === undefined) throw new Error('fixture source is missing') + source.outputs[0].satoshis++ + } + return params.map(verifyParamsInJavaScript) + } + } + const subject = coordinator(tracker, {}, verifier) + const mutated = subject.verify({ beef: firstEvidence, outputIndex: 0 }) + + await entered.promise + release.resolve() + await expectCode(mutated, 'invalid-evidence') + mutate = false + await expect(subject.verify({ beef: firstEvidence, outputIndex: 0 })).resolves.toMatchObject({ + outputIndex: 0 + }) + }) + + it('does not free a non-abortable backend slot until its cancelled work actually settles', async () => { + const { tracker, firstEvidence, secondEvidence } = await sharedAncestorFixture() + const entered = deferred() + const release = deferred() + let calls = 0 + const verifier: BdkVerifierInterface = { + supportsMemoryLimit: true, + verifyScripts: async () => false, + verifyScriptsBatch: async params => { + calls++ + entered.resolve() + await release.promise + return params.map(verifyParamsInJavaScript) + } + } + const subject = coordinator(tracker, { concurrentTransactions: 1 }, verifier) + const abort = new AbortController() + const cancelled = subject.verify( + { beef: firstEvidence, outputIndex: 0 }, + { signal: abort.signal } + ) + + await entered.promise + abort.abort() + await expectCode(cancelled, 'cancelled') + const pending = subject.verify({ beef: secondEvidence, outputIndex: 0 }) + expect(subject.getStats()).toMatchObject({ pendingTransactions: 1, activeAttempts: 1 }) + expect(calls).toBe(1) + release.resolve() + await expect(pending).resolves.toMatchObject({ outputIndex: 0 }) + expect(calls).toBeGreaterThanOrEqual(2) + }) + + it('preflights approved duplicate and conflicting ancestry before warmed script work can be reused', async () => { + const { tracker, ancestor, firstEvidence, secondEvidence } = await sharedAncestorFixture() + const confirmed = ancestor.inputs[0].sourceTransaction + if (confirmed === undefined) throw new Error('fixture root is missing') + const key = new PrivateKey(43) + const p2pkh = new P2PKH() + const subject = coordinator(tracker) + const verify = jest.spyOn(Transaction.prototype, 'verify') + const validate = jest.spyOn(Spend.prototype, 'validateJavaScript') + await subject.verify({ beef: firstEvidence, outputIndex: 0 }) + const warmedVerifyCalls = verify.mock.calls.length + const warmedScriptCalls = validate.mock.calls.length + + const duplicate = new Transaction() + for (let index = 0; index < 2; index++) { + duplicate.addInput({ + sourceTransaction: confirmed, + sourceOutputIndex: 0, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + } + duplicate.addOutput({ satoshis: 19, lockingScript: p2pkh.lock(key.toAddress()) }) + await duplicate.sign() + + const createConflictingParent = async (satoshis: number): Promise => { + const parent = new Transaction() + parent.addInput({ + sourceTransaction: confirmed, + sourceOutputIndex: 0, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + parent.addOutput({ satoshis, lockingScript: p2pkh.lock(key.toAddress()) }) + await parent.sign() + return parent + } + const left = await createConflictingParent(9) + const right = await createConflictingParent(8) + const joined = new Transaction() + for (const sourceTransaction of [left, right]) { + joined.addInput({ + sourceTransaction, + sourceOutputIndex: 0, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + } + joined.addOutput({ satoshis: 15, lockingScript: p2pkh.lock(key.toAddress()) }) + await joined.sign() + + await expectCode( + subject.verify({ beef: duplicate.toBEEF(), outputIndex: 0 }), + 'invalid-evidence' + ) + await expectCode(subject.verify({ beef: joined.toBEEF(), outputIndex: 0 }), 'invalid-evidence') + expect(verify).toHaveBeenCalledTimes(warmedVerifyCalls) + expect(validate).toHaveBeenCalledTimes(warmedScriptCalls) + await subject.verify({ beef: secondEvidence, outputIndex: 0 }) + expect(validate).toHaveBeenCalledTimes(warmedScriptCalls + 1) + verify.mockRestore() + validate.mockRestore() + }) + + it('rejects changed source bytes before a warmed target cache can be selected by a txid hint', async () => { + const { tracker, tx, evidence } = await fixture() + const subject = coordinator(tracker) + const verify = jest.spyOn(Transaction.prototype, 'verify') + await subject.verify({ beef: evidence, outputIndex: 0 }) + const altered = Transaction.fromBEEF(evidence) + const source = altered.inputs[0].sourceTransaction + if (source === undefined) throw new Error('fixture source is missing') + source.outputs[0].satoshis++ + + await expectCode( + subject.verify({ beef: altered.toBEEF(), outputIndex: 0, txid: tx.id('hex') }), + 'invalid-evidence' + ) + expect(verify).toHaveBeenCalledTimes(1) + await subject.verify({ beef: evidence, outputIndex: 0, txid: tx.id('hex') }) + expect(verify).toHaveBeenCalledTimes(1) + verify.mockRestore() + }) + + it('bounds candidates, complete graph shape, pending chain calls, and independent contexts', async () => { + const { tracker, evidence } = await fixture() + await expectCode( + coordinator(tracker, { transactions: 1 }).verify({ beef: evidence, outputIndex: 0 }), + 'limit' + ) + await expectCode( + coordinator(tracker, { inputs: 1 }).verify({ beef: evidence, outputIndex: 0 }), + 'limit' + ) + await expectCode( + coordinator(tracker, { scriptBytes: 1 }).verify({ beef: evidence, outputIndex: 0 }), + 'limit' + ) + + const release = deferred() + const entered = deferred() + tracker.gate = release.promise + tracker.onRootCall = () => entered.resolve() + const candidateLimited = coordinator(tracker, { candidatesPerTransaction: 8 }) + const controllers = Array.from({ length: 8 }, () => new AbortController()) + const requests = controllers.map((controller, index) => + candidateLimited.verify( + { beef: index === 0 ? evidence : alternateReceipt(evidence, index), outputIndex: 0 }, + { signal: controller.signal } + ) + ) + await entered.promise + await expectCode( + candidateLimited.verify({ beef: alternateReceipt(evidence, 9), outputIndex: 0 }), + 'limit' + ) + controllers.forEach(controller => controller.abort()) + await Promise.all(requests.map(async request => await expectCode(request, 'cancelled'))) + release.resolve() + + const second = await sharedAncestorFixture() + for (const root of second.tracker.roots) tracker.roots.add(root) + const chainRelease = deferred() + const chainEntered = deferred() + tracker.gate = chainRelease.promise + tracker.onRootCall = () => chainEntered.resolve() + const chainLimited = coordinator(tracker, { pendingChainCalls: 1 }) + const firstRequest = chainLimited.verify({ beef: evidence, outputIndex: 0 }) + await chainEntered.promise + await expectCode(chainLimited.verify({ beef: second.firstEvidence, outputIndex: 0 }), 'limit') + expect(chainLimited.getStats().pendingChainCalls).toBe(1) + chainRelease.resolve() + await expect(firstRequest).resolves.toMatchObject({ outputIndex: 0 }) + + const contexts = coordinator(tracker) + const contextVerify = jest.spyOn(Transaction.prototype, 'verify') + await contexts.verify({ beef: evidence, outputIndex: 0 }) + const replacement = new LocalChainTracker() + for (const root of tracker.roots) replacement.roots.add(root) + contexts.setContext({ + chainTracker: replacement, + chainNamespace: 'independent-network', + policyId: 'independent-policy' + }) + await contexts.verify({ beef: evidence, outputIndex: 0 }) + expect(contextVerify).toHaveBeenCalledTimes(2) + contextVerify.mockRestore() + }) + + it('rejects a two-height result when its token changes after the final H200 check and before H100 completes', async () => { + const { tracker, evidence, roots } = await twoAnchorFixture() + tracker.onAnchorCheck = (blockHeight, count) => { + if (blockHeight === 200 && count === 2) { + tracker.removeRoot(roots[0], 200) + tracker.token = 'tip-b' + } + } + const subject = coordinator(tracker) + + await expectCode(subject.verify({ beef: evidence, outputIndex: 0 }), 'context-changed') + expect(tracker.rootChecks.get(200)).toBe(2) + expect(tracker.rootChecks.get(100)).toBe(2) + expect(subject.getStats().cachedTransactions).toBe(0) + }) + + it('rejects an in-flight same-height root when the trusted token changes', async () => { + const { tracker, evidence } = await twoAnchorFixture() + const release = deferred() + const entered = deferred() + tracker.gate = release.promise + tracker.onRootCall = () => entered.resolve() + const subject = coordinator(tracker) + const request = subject.verify({ beef: evidence, outputIndex: 0 }) + + await entered.promise + tracker.token = 'tip-b' + release.resolve() + await expectCode(request, 'context-changed') + expect(subject.getStats().cachedTransactions).toBe(0) + }) + + it('keeps cached crypto work when a new canonical tip is already stable before the read begins', async () => { + const { tracker, evidence } = await twoAnchorFixture() + const verify = jest.spyOn(Transaction.prototype, 'verify') + const subject = coordinator(tracker) + + await subject.verify({ beef: evidence, outputIndex: 0 }) + tracker.token = 'tip-b' + await expect(subject.verify({ beef: evidence, outputIndex: 0 })).resolves.toMatchObject({ + outputIndex: 0 + }) + expect(verify).toHaveBeenCalledTimes(1) + expect(tracker.tokenCalls).toBeGreaterThanOrEqual(4) + verify.mockRestore() + }) + + it('fails closed when token acquisition fails, then permits a clean later recovery', async () => { + const { tracker, evidence } = await twoAnchorFixture() + tracker.tokenFailure = new Error('trusted provider unavailable') + const subject = coordinator(tracker) + + await expectCode(subject.verify({ beef: evidence, outputIndex: 0 }), 'invalid-evidence') + tracker.tokenFailure = undefined + await expect(subject.verify({ beef: evidence, outputIndex: 0 })).resolves.toMatchObject({ + outputIndex: 0 + }) + }) + + it('keeps token I/O counted after cancellation and never publishes its late completion', async () => { + const { tracker, evidence } = await twoAnchorFixture() + const release = deferred() + const entered = deferred() + tracker.tokenGate = release.promise + const originalToken = tracker.getVerificationContextToken.bind(tracker) + tracker.getVerificationContextToken = async signal => { + entered.resolve() + return await originalToken(signal) + } + const subject = coordinator(tracker) + const abort = new AbortController() + const request = subject.verify({ beef: evidence, outputIndex: 0 }, { signal: abort.signal }) + + await entered.promise + abort.abort() + await expectCode(request, 'cancelled') + expect(subject.getStats().pendingChainCalls).toBe(1) + release.resolve() + await new Promise(resolve => setImmediate(resolve)) + expect(tracker.tokenAborts).toBe(1) + expect(subject.getStats()).toMatchObject({ cachedTransactions: 0, pendingChainCalls: 0 }) + }) + + it('rejects a shouldVerifyScripts mutation before publication and accepts a clean retry', async () => { + const { tracker, firstEvidence } = await sharedAncestorFixture() + let mutate = true + const verifier: BdkVerifierInterface = { + supportsMemoryLimit: true, + shouldVerifyScripts: params => { + if (mutate) { + const source = params.tx.inputs[0]?.sourceTransaction + if (source === undefined) throw new Error('fixture source is missing') + source.outputs[0].satoshis++ + } + return true + }, + verifyScripts: async () => true + } + const subject = coordinator(tracker, {}, verifier) + + await expectCode(subject.verify({ beef: firstEvidence, outputIndex: 0 }), 'invalid-evidence') + mutate = false + await expect(subject.verify({ beef: firstEvidence, outputIndex: 0 })).resolves.toMatchObject({ + outputIndex: 0 + }) + }) + + it.each([ + ['the JavaScript interpreter', undefined], + [ + 'a single backend call', + { + supportsMemoryLimit: true, + verifyScripts: async () => { + throw new ScriptResourceLimitError('stack', 1, 2) + } + } satisfies BdkVerifierInterface + ], + [ + 'a batch backend call', + { + supportsMemoryLimit: true, + verifyScripts: async () => true, + verifyScriptsBatch: async () => { + throw new ScriptResourceLimitError('stack', 1, 2) + } + } satisfies BdkVerifierInterface + ] + ])( + 'maps script resource exhaustion from %s to the local limit error', + async (_path, verifier) => { + const { tracker, evidence } = await fixture() + const subject = coordinator(tracker, { scriptMemoryBytes: 1 }, verifier) + await expectCode(subject.verify({ beef: evidence, outputIndex: 0 }), 'limit') + } + ) +}) diff --git a/packages/sdk/src/transaction/index.ts b/packages/sdk/src/transaction/index.ts index 9950f03ef..accd1dbcd 100644 --- a/packages/sdk/src/transaction/index.ts +++ b/packages/sdk/src/transaction/index.ts @@ -1,16 +1,30 @@ export { default as Transaction } from './Transaction.js' -export type { default as BdkVerifierInterface, BdkVerifyScriptsParams } from './BdkVerifierInterface.js' +export type { + default as BdkVerifierInterface, + BdkVerifyScriptsParams +} from './BdkVerifierInterface.js' export * from './ScriptVerificationBackend.js' export { default as MerklePath } from './MerklePath.js' export type { default as TransactionInput } from './TransactionInput.js' export type { default as TransactionOutput } from './TransactionOutput.js' -export type { - Broadcaster, - BroadcastFailure, - BroadcastResponse -} from './Broadcaster.js' +export type { Broadcaster, BroadcastFailure, BroadcastResponse } from './Broadcaster.js' export { isBroadcastResponse, isBroadcastFailure } from './Broadcaster.js' export type { default as ChainTracker } from './ChainTracker.js' +export { TransactionEvidenceCoordinator } from './TransactionEvidenceCoordinator.js' +export type { + TransactionEvidenceContext, + TransactionEvidenceCoordinatorOptions +} from './TransactionEvidenceCoordinator.js' +export { + TransactionEvidenceError, + defaultTransactionEvidenceLimits +} from './TransactionEvidence.js' +export type { + TransactionEvidence, + VerifiedTransactionOutput, + TransactionEvidenceLimits, + TransactionEvidenceErrorCode +} from './TransactionEvidence.js' export { default as BeefTx } from './BeefTx.js' export * from './Beef.js' export { default as BeefParty } from './BeefParty.js' diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index a2ff7a339..c0a8cf2ff 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -7,15 +7,18 @@ attention to changes that materially alter behavior or extend functionality. ## wallet-toolbox (unreleased) - Secure final overlay identity discovery with independently verified transaction - evidence, byte-derived txid/hint checks, output bounds, the subject-signed - identity envelope, and explicit certificate success. Cached overlay responses - retain evidence only and repeat verification before returning identities. - Configure the existing wallet Services ChainTracker; direct `identityUtils` - callers must pass it explicitly. Local contacts, permission handling, wallet - RPC shapes, and current pagination behavior are retained. This security fix is - a patch in the full, browser, and mobile packages (2.11.1). No storage migration - is required. See [identity verification](docs/identity-verification.md) for the - supported token contract and remaining chain-cache/discovery limits. + evidence, byte-derived txid/hint checks, complete unconfirmed-graph + consistency, bounded resolver intake, the standard subject-signed identity + envelope, and explicit certificate success. C02/C03 add the exported SDK + evidence coordinator, optional chain-provider context markers, bounded + transaction/script work sharing, and revalidation of canonical anchors on + positive-cache use. Cached overlay receipts and decrypted certificates remain + wallet-owned, bounded, and separate from local contacts and trust policy. + Existing permission negotiation, pagination, wallet RPC shapes, and resolver + call forms remain characterized as compatible. See [identity verification](docs/identity-verification.md) + for the current contracts and limits. Package and packed-consumer validation + for the current source remains pending release review; historical C01 bundle + measurements are retained in that guide. - Keep Argon2id-backed UMP v3 wallets available in React Native and other runtimes without WebAssembly by falling back to an asynchronously yielding, diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index e140f2c1b..92f1fbd7b 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -426,14 +426,19 @@ does not apply schema changes. ### Overlay identity verification -Final identity discovery verifies transaction evidence against the wallet's -existing `Services.getChainTracker()` configuration before validating the -subject-signed identity output and certificate. Cached responses are reverified -before use. Direct `identityUtils` callers must supply a canonical `ChainTracker`; -missing context or invalid evidence produces no overlay identities. Local -contacts retain their separate policy. Inclusion does not establish unspentness -or freshness. See [identity verification](docs/identity-verification.md) for -configuration, compatibility, and remaining verification lifecycle limits. +Final identity discovery copies bounded resolver receipts, verifies their +transaction graph and canonical anchors with the wallet's existing +`Services.getChainTracker()`, and then validates the standard subject-signed +identity envelope and certificate. Transaction and certificate reuse remain +bounded and canonical evidence is rechecked before cached results are used; +fresh provider tokens bracket asynchronous anchor checks where the configured +tracker supplies them. Invalid candidate evidence is dropped, while typed +limit/timeout outcomes propagate to the caller. +Direct `identityUtils` callers must supply a canonical `ChainTracker`; missing +context or invalid evidence produces no overlay identities. Local contacts +retain their separate policy. Inclusion does not establish unspentness or +freshness. See [identity verification](docs/identity-verification.md) for +current C01/C02/C03 contracts, compatibility characterization, and limits. ```bash git clone https://github.com/bsv-blockchain/ts-stack.git diff --git a/packages/wallet/wallet-toolbox/client/README.md b/packages/wallet/wallet-toolbox/client/README.md index de07e7257..a3f935f25 100644 --- a/packages/wallet/wallet-toolbox/client/README.md +++ b/packages/wallet/wallet-toolbox/client/README.md @@ -126,12 +126,14 @@ CSP is an application and deployment concern rather than a package-level access ## Contributor checks -Final overlay identity discovery uses the configured wallet Services ChainTracker -to verify transaction evidence before returning certificate results, including -on response-cache hits. Local contacts retain their separate policy. See the -[identity verification guide](../docs/identity-verification.md) for configuration, -compatibility, and chain-cache limits; inclusion is not proof of freshness or -unspentness. +Final overlay identity discovery copies bounded resolver receipts, verifies the +complete transaction graph and canonical anchors with the configured wallet +Services ChainTracker, and validates the standard subject-signed certificate +envelope. Cached transaction evidence is rechecked before use; local contacts +retain their separate policy. See the [identity verification guide](../docs/identity-verification.md) +for C01/C02/C03 compatibility characterization and limits. Inclusion is not +proof of freshness or unspentness. Current package and packed-consumer +validation remains pending the release review. From the repository root, build the SDK and package before running the installed-consumer browser gate: diff --git a/packages/wallet/wallet-toolbox/docs/identity-verification.md b/packages/wallet/wallet-toolbox/docs/identity-verification.md index 40d036a81..fc2ddc437 100644 --- a/packages/wallet/wallet-toolbox/docs/identity-verification.md +++ b/packages/wallet/wallet-toolbox/docs/identity-verification.md @@ -1,106 +1,194 @@ # Identity overlay verification -This page records the bounded C01 source contract for identity results returned by the wallet toolbox. It is an inventory of the current implementation and its compatibility edges. It does not describe completion of the wider overlay reliability plan. +This page records the current C01, C02, and C03 source contract for identity +results returned by Wallet Toolbox. The implementation and its tests remain +under review; this page is not a claim that the wider verification plan or +release gates have completed. ## Evidence boundary -An overlay lookup result is a host-supplied `LookupAnswer`. The resolver permits an output to carry a transaction id hint, but the hint is metadata rather than chain authority. C01 treats each output as untrusted evidence and verifies the BEEF bytes and selected output with the wallet's configured `ChainTracker` before decoding an identity certificate. - -The shared transaction seam is [`verifyOverlayOutput`](../src/utility/verifyOverlayOutput.ts). It owns a copy of the BEEF bytes, derives the transaction id from those bytes, and rejects a supplied hint unless it matches that derived id. It checks the selected output and verifies the transaction graph through the caller's tracker. An unconfirmed branch must have ancestry; an input-free unconfirmed leaf cannot anchor an identity result. The preflight also rejects duplicate inputs and conflicting spends across distinct unconfirmed ancestors, while allowing a shared transaction to be traversed once. This is consistency of the supplied graph, not an external unspentness check. The generic helper has no network default, certificate policy, verdict cache, or identity-specific locking-key rule. - -The wallet obtains the tracker through [`WalletServicesOptions.chainTracker`](../src/sdk/WalletServices.interfaces.ts) and [`Services.getChainTracker`](../src/services/Services.ts). A configured tracker is the wallet's chain authority. The utility functions [`queryOverlay`](../src/utility/identityUtils.ts), [`parseResults`](../src/utility/identityUtils.ts), and [`parseResults$`](../src/utility/identityUtils.ts) accept an explicit optional tracker; when it is missing they fail closed with an empty result. This keeps a caller from accidentally treating an overlay response as verified. - -The identity parser then decodes the selected output as PushDrop, requires the subject-signed field payload to verify, requires `VerifiableCertificate.verify()` to return `true`, decrypts the public keyring, and requires nonempty decrypted fields. A cryptographic signature failure is already surfaced as an exception by [`ProtoWallet.verifySignature`](../../../sdk/src/wallet/ProtoWallet.ts), which the parser drops with the candidate. The earlier ignored boolean from certificate verification is therefore not an invalid-certificate bypass; C01 makes the success condition explicit while adding the missing transaction evidence checks. Candidate errors are not logged because parser/decryption exceptions may contain identity data. - -## Standard identity envelope - -The interoperable envelope is the one emitted by the default [`IdentityClientOptions`](../../../sdk/src/identity/types/index.ts): protocol ID `[1, 'identity']`, key ID `'1'`, token amount `1`, and output index `0`. [`IdentityClient.publiclyRevealAttributes`](../../../sdk/src/identity/IdentityClient.ts) creates the PushDrop output and broadcasts it to `tm_identity`. - -The current [`IdentityTopicManager`](../../../../packages/overlays/topics/src/identity/IdentityTopicManager.ts) verifies the same subject-signed PushDrop fields with protocol `[1, 'identity']` and key ID `'1'`, then checks certificate validity and nonempty decrypted attributes. C01 matches that actual server contract. `IdentityClientOptions` exposes custom protocol and key values, but the current topic manager does not accept arbitrary values: a non-default protocol ID or key ID is a compatibility hazard and can cause the topic to reject the output. C01 does not widen topic acceptance. - -There is no existing topic contract for comparing an identity certificate to an arbitrary `lockingPublicKey`, so C01 adds no such equality rule. The subject-signed envelope, certificate signature, successful decryption, and trusted-certifier policy remain the relevant checks. +An overlay result is a host-supplied `LookupAnswer`. A host TXID hint and +`context` are metadata, never chain authority. [`queryOverlayEvidence`](../src/utility/identityUtils.ts) +copies the BEEF and context bytes received through the additive resolver +`onEvidence` callback. Resolver callback intake defaults to 512 outputs and +16 MiB per query and accepts `evidenceLimits: { maxOutputs?, maxBytes? }` as +configurable local policy. It uses a 300 ms identity grace window. Older or +custom resolvers that do not invoke the callback remain usable through the +legacy output-list fallback. + +Direct `queryOverlayEvidence` callers can configure +`{ candidateBytes?, retainedBytes?, outputs? }`. To admit larger valid +evidence, pass matching byte limits to both this intake option and the third +`IdentityEvidenceVerifier` constructor argument (`TransactionEvidenceLimits`). +The resolver and verifier limits are separate admission boundaries and are not +consensus limits. + +[`TransactionEvidenceCoordinator`](../../../sdk/src/transaction/TransactionEvidenceCoordinator.ts) +parses the owned bytes, derives the selected transaction ID, compares any +supplied hint, checks the selected output, validates the complete unconfirmed +graph, and performs fresh canonical-root checks through the caller's +`ChainTracker`. It checks source transaction/value/script bindings, source TXID +consistency, duplicate spent outpoints, input/script limits, and rejects an +unconfirmed zero-input leaf. A successful result identifies a transaction +output and locking script; it does not establish service relevance, current +unspentness, ownership, or freshness. + +The coordinator's `chainNamespace` and `policyId` are explicit caller-owned +trust context. An optional synchronous +`ChainTracker.getVerificationContext(): string | number` is a provider, +policy, or recovery-generation marker. The optional +`getVerificationContextToken(signal?)` obtains a fresh trusted tip/context +token and brackets canonical-root and height checks. Chaintracks and local +adapters include monotonic reorganization/reset epochs and fence reset entry +and failure paths. `LocalChainTracker` tokens cover only the providers +participating in the current attempt: remote-only uses fallback identity, +local-primary uses local identity. An unused provider cannot stand in, and a +missing participating identity fails closed. Built-in remote ChainTracks +clients advertise reorg-event capability explicitly; method presence is not +capability. `Services.getChainTracker()` publishes one coalesced wrapper +before yielding. A remote tip equality read cannot detect an unobserved ABA +transition, and a token over multiple sources is not an atomic multi-source +snapshot. Canonical roots and observed heights remain dependencies that are +rechecked on each use, including positive-cache reuse. Existing trackers may +omit either optional method and may ignore optional abort signals. + +`EvidenceScriptWork` in the SDK now shares bounded script work by a binding that +covers the actual transaction, all input source-output bytes and values, and +script policy/backend parameters. The binding is transaction-wide rather than +an input-index-only key. A cache hit can skip cryptographic script execution +only. It still performs source/value binding, complete graph traversal, +canonical chain calls, and policy/context checks. Non-batch individual +executions preserve each entry's fulfilled or rejected result so a failed +sibling cannot reject a successful shared ancestor. Only a backend batch-level +failure rejects the whole batch. Non-abortable backend work remains counted +until its actual promise settles, and in-flight owners are isolated. A normal +block arrival before the next reuse does not erase exact cryptographic script +work when its transaction/source/policy binding remains valid; canonical roots +and fresh context are still checked. + +## Identity envelope + +The interoperable identity output is the standard subject-signed PushDrop +created by [`IdentityClient.publiclyRevealAttributes`](../../../sdk/src/identity/IdentityClient.ts): +protocol ID `[1, 'identity']`, key ID `'1'`, token amount `1`, and output index +`0`. [`IdentityTopicManager`](../../../../packages/overlays/topics/src/identity/IdentityTopicManager.ts) +checks the same protocol and key, then certificate validity and nonempty +attributes. `IdentityClientOptions` exposes custom protocol and key values, but +the current server topic does not accept arbitrary values; custom values remain +a compatibility characterization item and C01 does not widen server +acceptance. + +The parser checks the subject signature, requires `VerifiableCertificate.verify()` +to return `true`, decrypts the fields, and requires nonempty decrypted fields. +`ProtoWallet.verifySignature` throws on a cryptographic failure today; the +parser catches and rejects that candidate. C01 does not claim an invalid +certificate can bypass this throw. There is no existing topic contract for an +arbitrary `lockingPublicKey` equality rule, so none is added. ## Cache and contact boundaries -The wallet's overlay evidence cache is a two-minute response cache in [`Wallet.ts`](../src/Wallet.ts). Cached BEEF is revalidated on every use with the current tracker and certificate checks. If any candidate is rejected, the query entry is evicted so a later call can fetch fresh evidence. The cache is not a chain verdict and does not change local contact behavior. - -Local contacts are a separate wallet-owned source. [`Wallet.ts`](../src/Wallet.ts) synthesizes contact results with the local contact's subject/certifier relationship and local trust data. A contact hit may short-circuit the overlay path as before; it must retain local-contact provenance and must not be presented as an overlay SPV result. Contact lookup failures fall through to the network path. - -## Compatibility inventory and limits - -The public identity path spans the SDK wallet interfaces and clients, JSON and binary wallet transports, toolbox wallet managers, and [`IdentityClient`](../../../sdk/src/identity/IdentityClient.ts). C01 preserves those Promise method shapes and the `parseResults$` async-iterable shape. It does not change permission negotiation or pagination. The interface documentation says `seekPermission` defaults true, while the validator currently applies a false default; see [`Wallet.interfaces.ts`](../../../sdk/src/wallet/Wallet.interfaces.ts) and [`validationHelpers.ts`](../../../sdk/src/wallet/validationHelpers.ts). The wallet's current overlay calls also do not forward validated `limit` and `offset`, although the identity lookup service accepts them. These are compatibility characterization items for W00/W02, outside this C01 document. - -The resolver currently validates only the shape of a txid hint in [`LookupResolver.ts`](../../../sdk/src/overlay-tools/LookupResolver.ts). Its aggregation deduplicates by the hinted or derived txid and output index, keeping the first answer, and its fast path trusts a nonempty hint ([`LookupResolver.ts`](../../../sdk/src/overlay-tools/LookupResolver.ts), [`LookupResolver.ts`](../../../sdk/src/overlay-tools/LookupResolver.ts)). C02 owns the first-wins raw resolver suppression and pending full txid sharing work. C01 verifies the evidence that reaches the wallet; it cannot recover an alternate candidate discarded before parsing. - -The existing `ChaintracksChainTracker` still has its height-keyed root-cache and reorganization lifecycle. C01 makes no reorganization-safety claim; C03 covers that Chaintracks cache limitation and lifecycle work. C01 adds no workers, shared transaction jobs, response/graph budgets, or whole-attempt deadline. Large proofs still incur parsing and verification work on the calling runtime. These limits and canonical-context fencing require the later coordinator/runtime slices; no latency, reorganization safety, or deployment claim is made here. - -## Requirement and verification mapping - -The scoped source work supports requirement V1 (independent transaction evidence), the byte-binding portion of V2 (BEEF-derived txid must match any hint), and V5 (certificate success and trust policy). The planned characterization cases are T04 (false txid hint), T07 (confirmed/unconfirmed ancestry, scripts, values, and graph-internal conflicts), T09 (output/envelope/certificate validity), T10 (permission, cache, and local contacts), and T13 (Promise and pagination compatibility). This mapping is evidence for the C01 slice only; it is not a claim that the full verification plan has passed. - -## Configuration and migration - -This is a patch security correction in the 2.11.1 full, browser, and mobile -packages; the aggregate release-note candidate remains minor relative to the -recorded 2.10.4 published baseline. Wallet RPC and stored data need no migration. -Wallet builders keep their existing `Services` chain configuration. Its -`getChainTracker()` selects `options.chainTracker`, or wraps the configured -`options.chaintracks`; a configuration/availability error never becomes an -acceptance verdict. Use a chain source maintained independently of overlay hosts. - -Direct utility callers previously supplied only an answer. They must now pass -their canonical chain source: +The wallet keeps three distinct boundaries: + +- Transaction evidence is revalidated through the current chain tracker on + every use. Coordinator positive results are bounded by configurable generic + SDK limits, including 128 entries, 16 MiB retained bytes, and 60 seconds. +- Decrypted certificate JSON is wallet-owned and copied on return. Its cache is + bounded at 2 MiB, 128 entries, and 60 seconds. +- Raw overlay lookup receipts are copied into a wallet cache bounded at 32 + queries, 16 MiB, and 120 seconds. A rejected or incomplete parse evicts that + query entry so a later call can fetch again. + +The wallet obtains the tracker from `getServices().getChainTracker()` and +reuses the configured instance. Direct [`queryOverlay`](../src/utility/identityUtils.ts), +`parseResults`, and `parseResults$` calls fail closed when no tracker is +provided. [`verifyOverlayOutput`](../src/utility/verifyOverlayOutput.ts) +requires an explicit tracker. Local contacts remain a separate wallet-owned +source and retain their existing short-circuit and trust behavior; contact +results must retain local provenance and are not overlay SPV results. Invalid +candidate evidence is dropped, while typed `limit` and `timeout` outcomes +propagate to the wallet caller instead of becoming a definitive empty result. + +## Compatibility characterization + +C01, C02, and C03 preserve existing Promise and `parseResults$` async-iterable +shapes. The resolver's `onEvidence` callback is optional and additive. Legacy +resolver answers continue to aggregate by TXID/output index with first-wins +suppression; a host hint is only a resolver fast path and must be re-derived +and checked by security-sensitive consumers. The callback is the bounded path +for consumers that need each host receipt before that suppression. Existing +2-second lookup and 5-second tracker-wait defaults remain unchanged. + +Permission negotiation and pagination are unchanged. The interface +characterization still records that documentation says `seekPermission` +defaults to true while the validator currently applies false. Identity lookup +also retains the existing limit/offset validation and forwarding +characterization; this evidence work does not silently change either contract. + +The standard topic envelope characterization and custom `IdentityClientOptions` +behavior above remain required for interoperability. No arbitrary locking-key +equality rule is implied. Chaintracks and local adapters now expose monotonic +reorganization/reset epochs and fence reset lifecycle races, including reset +ownership checks after disposal and before a destructive hook. Remote tip +equality still cannot detect an unobserved ABA transition, and no adapter token +provides an atomic multi-source snapshot. C03 therefore makes no general +reorganization-safety claim. + +## Usage + +Wallet callers use the existing Services configuration: ```ts const tracker = await wallet.getServices().getChainTracker() const certificates = await parseResults(answer, tracker) ``` -Omitting the optional argument remains source-compatible but fails closed. -There is no bypass toggle. Failed candidate evidence evicts a wallet response -cache entry; failures never establish a permanent negative verdict for a txid. -Untrusted cached bytes may be shared within this wallet, but decrypted result -objects are rebuilt on each call and then passed through the existing trust -settings. The existing two-minute trust-settings snapshot policy is unchanged. +Omitting the utility tracker remains source-compatible but returns no verified +overlay identities. There is no bypass toggle. Wallet RPC shapes, stored data, +permission defaults, and pagination behavior require no migration from this +source work. + +On browser and React Native runtimes, `parseResults$` cooperatively yields +between certificates. This is a current-runtime scheduling behavior, with no +worker, throughput, latency, resource-isolation, or deployment guarantee. +C04/C05 work and any whole-plan completion claim remain future scope. + +## Requirement and verification mapping + +The scoped source supports V1 (independent transaction evidence), the +byte-binding portion of V2 (BEEF-derived TXID versus any hint), and V5 +(certificate success and trust policy). T04, T07, T09, T10, and T13 remain the +relevant characterization cases for false hints, graph and anchor checks, +envelope/certificate validity, permissions/cache/contacts, and Promise/ +pagination compatibility. The synthetic shared-ancestor fixture is 556 BEEF +bytes with three reachable transactions, three inputs, and 314 serialized +script bytes; two child graphs produce three actual script executions. Current +tests are evidence for these slices only; validation is not final until the +required package and consumer checks pass. -## Package size review +## Historical C01 bundle measurements -On 8 September 2026, the originating review task -`01a081b5-26d4-7ad1-8d85-243fe238d595` explicitly approved these measured C01 -budget adjustments under -[`governance/browser-artifact-policy.json`](../../../../governance/browser-artifact-policy.json). -The policy requires a versioned source change, composition evidence, and explicit -review. All three published artifacts advance from 2.11.0 to 2.11.1. Mandatory -transaction, graph, and identity checks remain in the portable bundles. +The following measurements are retained as historical C01 evidence for the +2.11.1 artifacts. They are measurements only, not passing platform gates and +not a release decision. C02/C03 source changes require fresh package and +consumer validation. Measurements used Node 24.15.0 and pnpm 10.33.2 on the same macOS host, with base commit `2bc799a8d8e535242e6de2d305f426ce3975ea7b` extracted into a temporary -source tree and built against the same unchanged SDK and dependency graph. -`pnpm build` ran in each base/current client and mobile package. A temporary -copy of the platform checker printed every size instead of evaluating budgets: -`node /tmp/c01-measure-baseline.mjs browser`, -`node /tmp/c01-measure-baseline.mjs mobile`, -`node /tmp/c01-measure-platform.mjs browser`, and -`node /tmp/c01-measure-platform.mjs mobile`. -These were **measurements only, not passing platform gates**. The original -platform checker was not modified. +source tree and the same SDK/dependency graph. A temporary copy of the +platform checker printed each size instead of evaluating budgets. The original +checker was not modified. Each cell lists raw / gzip / Brotli bytes: -| Consumer | Base | C01 | Reviewed maximum | -| -------- | --------------------------------- | --------------------------------- | --------------------------------- | -| Vite | 1,692,309 / 399,380 / 312,207 | 1,694,805 / 400,062 / 312,426 | 1,696,000 / 401,000 / 314,000 | -| esbuild | 1,320,184 / 363,792 / 291,500 | 1,322,211 / 364,400 / 291,926 | 1,324,000 / 365,000 / 293,000 | -| Metro | 1,747,262 / 443,100 / 343,780 | 1,749,640 / 443,811 / 343,927 | 1,751,000 / 455,000 / 360,000 | +| Consumer | Base | C01 | Reviewed maximum | +| -------- | --------------------------------: | --------------------------------: | --------------------------------: | +| Vite | 1,692,309 / 399,380 / 312,207 | 1,694,805 / 400,062 / 312,426 | 1,696,000 / 401,000 / 314,000 | +| esbuild | 1,320,184 / 363,792 / 291,500 | 1,322,211 / 364,400 / 291,926 | 1,324,000 / 365,000 / 293,000 | +| Metro | 1,747,262 / 443,100 / 343,780 | 1,749,640 / 443,811 / 343,927 | 1,751,000 / 455,000 / 360,000 | | Hermes | 3,544,570 / 1,440,174 / 1,117,759 | 3,550,004 / 1,442,134 / 1,120,813 | 3,553,000 / 1,443,000 / 1,123,000 | -Raw growth is 2,496 / 2,027 / 2,378 / 5,434 bytes respectively (about -0.14–0.15%). Vite composition retains 106 modules and the same packages: -`@bsv/sdk`, `@bsv/wallet-toolbox-client`, `@noble/hashes`, `hash-wasm`, and `idb`. -The esbuild module count remains 173. No new dependency or platform-only import -was added. Shared wallet cache/fetch logic removes duplication; independent -verification and subject-envelope checks account for the added code. Only -exceeded dimensions changed, rounded to preserve comparable existing margins; -all other limits stay fixed. This is a reviewed security-feature payload change, -not an analysis exception. Original `test:browser` and `test:mobile` commands -remain the executable gates and are recorded separately in the delivery ledger. +The historical C01 result measured raw growth of 2,496 / 2,027 / 2,378 / +5,434 bytes for Vite, esbuild, Metro, and Hermes respectively. The C01 +composition retained the same dependencies and did not add a platform-only +import. The original `test:browser` and `test:mobile` commands remain the +executable gates for any current release evidence. diff --git a/packages/wallet/wallet-toolbox/mobile/README.md b/packages/wallet/wallet-toolbox/mobile/README.md index 3dab43aa3..6fcfab919 100644 --- a/packages/wallet/wallet-toolbox/mobile/README.md +++ b/packages/wallet/wallet-toolbox/mobile/README.md @@ -85,12 +85,14 @@ Capability negotiation fails before prefunding against an older server. See ## Use cases -Final overlay identity discovery uses the configured wallet Services ChainTracker -to verify transaction evidence before returning certificate results, including -on response-cache hits. Local contacts retain their separate policy. See the -[identity verification guide](../docs/identity-verification.md) for configuration, -compatibility, and chain-cache limits; inclusion is not proof of freshness or -unspentness. +Final overlay identity discovery copies bounded resolver receipts, verifies the +complete transaction graph and canonical anchors with the configured wallet +Services ChainTracker, and validates the standard subject-signed certificate +envelope. Cached transaction evidence is rechecked before use; local contacts +retain their separate policy. See the [identity verification guide](../docs/identity-verification.md) +for C01/C02/C03 compatibility characterization and limits. Inclusion is not +proof of freshness or unspentness. Current package and packed-consumer +validation remains pending the release review. ### Self-custody BSV wallet on a phone diff --git a/packages/wallet/wallet-toolbox/src/Wallet.ts b/packages/wallet/wallet-toolbox/src/Wallet.ts index 65a8cedaf..e2d84c06d 100644 --- a/packages/wallet/wallet-toolbox/src/Wallet.ts +++ b/packages/wallet/wallet-toolbox/src/Wallet.ts @@ -81,7 +81,12 @@ import { createAction, CreateActionResultX } from './signer/methods/createAction import { signAction, SignActionResultX } from './signer/methods/signAction' import { internalizeAction } from './signer/methods/internalizeAction' import { WalletSettingsManager } from './WalletSettingsManager' -import { parseResults, queryOverlayEvidence, transformVerifiableCertificatesWithTrust } from './utility/identityUtils' +import { + IdentityEvidenceVerifier, + parseResults, + queryOverlayEvidence, + transformVerifiableCertificatesWithTrust +} from './utility/identityUtils' import { maxPossibleSatoshis } from './storage/methods/generateChange' import { hasBrc177NoSendExpiryLabel, parseBrc177NoSendExpiryLabels } from './utility/brc177NoSendExpiry' import { createNoSendExpiryAction } from './signer/methods/createNoSendExpiryAction' @@ -365,6 +370,10 @@ export class Wallet implements WalletInterface, ProtoWallet { } async destroy(): Promise { + this._identityEvidenceClosed = true + this._identityEvidenceVerifier?.dispose() + this._overlayEvidenceCache.clear() + clearTimeout(this._overlayEvidenceExpiryTimer) await this.actionBatch.abort() await this.storage.destroy() if (this.privilegedKeyManager != null) this.privilegedKeyManager.destroyKey() @@ -793,8 +802,31 @@ export class Wallet implements WalletInterface, ProtoWallet { trustSettings: Awaited>['trustSettings'] } - /** Two-minute untrusted response cache. Every use repeats transaction and identity checks. */ - private readonly _overlayEvidenceCache = new Map() + /** Bounded two-minute untrusted receipts. Every use rechecks canonical evidence. */ + private readonly _overlayEvidenceCache = new Map() + private _overlayEvidenceExpiryTimer?: ReturnType + private _identityEvidenceVerifier?: IdentityEvidenceVerifier + private _identityEvidenceClosed = false + + private pruneOverlayEvidence(): void { + for (const [key, value] of this._overlayEvidenceCache) { + if (value.expiresAt <= Date.now()) this._overlayEvidenceCache.delete(key) + } + } + + private scheduleOverlayEvidenceExpiry(): void { + clearTimeout(this._overlayEvidenceExpiryTimer) + const expiresAt = Math.min(...[...this._overlayEvidenceCache.values()].map(value => value.expiresAt)) + if (!Number.isFinite(expiresAt)) return + this._overlayEvidenceExpiryTimer = setTimeout( + () => { + this.pruneOverlayEvidence() + this.scheduleOverlayEvidenceExpiry() + }, + Math.max(1, expiresAt - Date.now()) + ) + this._overlayEvidenceExpiryTimer.unref?.() + } private async discoverOverlayCertificates( query: unknown, @@ -804,17 +836,50 @@ export class Wallet implements WalletInterface, ProtoWallet { ): Promise { // Use the wallet's existing network/chain configuration, never the overlay host's verdict. const chainTracker = await this.getServices().getChainTracker() + if (this._identityEvidenceClosed) return [] + const chainNamespace = `wallet:${this.chain}` + if ( + this._identityEvidenceVerifier?.chainTracker !== chainTracker || + this._identityEvidenceVerifier.chainNamespace !== chainNamespace + ) { + this._identityEvidenceVerifier?.dispose() + this._identityEvidenceVerifier = new IdentityEvidenceVerifier(chainTracker, chainNamespace) + } + const verifier = this._identityEvidenceVerifier + this.pruneOverlayEvidence() let cached = forceRefresh ? undefined : this._overlayEvidenceCache.get(cacheKey) if (cached == null || cached.expiresAt <= now) { const value = await queryOverlayEvidence(query, this.lookupResolver) - cached = { value, expiresAt: now + 2 * 60 * 1000 } + if (this._identityEvidenceClosed) return [] + const bytes = + value.type === 'output-list' + ? value.outputs.reduce((total, output) => total + output.beef.length + (output.context?.length ?? 0), 0) + : 0 + cached = { value, bytes, expiresAt: now + 2 * 60 * 1000 } + this._overlayEvidenceCache.delete(cacheKey) + let retained = [...this._overlayEvidenceCache.values()].reduce((total, entry) => total + entry.bytes, 0) + while ( + this._overlayEvidenceCache.size > 0 && + (this._overlayEvidenceCache.size >= 32 || retained + bytes > 16 * 1024 * 1024) + ) { + const oldest = this._overlayEvidenceCache.keys().next().value! + retained -= this._overlayEvidenceCache.get(oldest)!.bytes + this._overlayEvidenceCache.delete(oldest) + } this._overlayEvidenceCache.set(cacheKey, cached) + this.scheduleOverlayEvidenceExpiry() } if (cached.value.type !== 'output-list') { this._overlayEvidenceCache.delete(cacheKey) return [] } - const certificates = await parseResults(cached.value, chainTracker) + let certificates: VerifiableCertificate[] + try { + certificates = await parseResults(cached.value, chainTracker, verifier) + } catch (error) { + this._overlayEvidenceCache.delete(cacheKey) + throw error + } // Failed evidence must allow another fetch, including after temporary chain unavailability. if (certificates.length !== cached.value.outputs.length) this._overlayEvidenceCache.delete(cacheKey) return certificates @@ -1098,10 +1163,7 @@ export class Wallet implements WalletInterface, ProtoWallet { if (vargs.labels.includes(specOpThrowReviewActions)) throwDummyReviewActions() if (hasBrc177NoSendExpiryLabel(vargs.labels)) { - throw new WERR_INVALID_PARAMETER( - 'labels', - 'BRC-177 noSend expiry labels only on outgoing createAction requests' - ) + throw new WERR_INVALID_PARAMETER('labels', 'BRC-177 noSend expiry labels only on outgoing createAction requests') } const r = await internalizeAction(this, auth, args) diff --git a/packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts b/packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts index 23ce78fcc..a42bd6102 100644 --- a/packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts +++ b/packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts @@ -1,4 +1,13 @@ -import { KeyDeriver, LookupAnswer, LookupResolver, PrivateKey, Validation } from '@bsv/sdk' +import { + KeyDeriver, + LookupAnswer, + LookupResolver, + MerklePath, + PrivateKey, + Transaction, + Validation, + VerifiableCertificate +} from '@bsv/sdk' import { Wallet } from '../Wallet' import { WalletSettingsManager } from '../WalletSettingsManager' import { WalletStorageManager } from '../storage/WalletStorageManager' @@ -8,7 +17,7 @@ import { IdentityVerificationFixture } from '../utility/__tests__/identityVerification.fixtures' -function walletFor(fixture: IdentityVerificationFixture) { +function walletFor(fixture: IdentityVerificationFixture, resolver?: LookupResolver) { const keyDeriver = new KeyDeriver(new PrivateKey(15)) const trustSettings = { trustLevel: 1, @@ -27,7 +36,7 @@ function walletFor(fixture: IdentityVerificationFixture) { keyDeriver, storage: new WalletStorageManager(keyDeriver.identityKey), services: { getChainTracker } as unknown as WalletServices, - lookupResolver: { query } as unknown as LookupResolver, + lookupResolver: resolver ?? ({ query } as unknown as LookupResolver), settingsManager: { get: getSettings } as unknown as WalletSettingsManager }) return { wallet, query, getChainTracker, getSettings, trustSettings } @@ -74,6 +83,51 @@ describe('Wallet final identity verification and compatibility', () => { expect(query).toHaveBeenCalledTimes(1) }) + it('recovers a valid alternate receipt before legacy resolver merging and keeps certificate/trust caches isolated', async () => { + const badTransaction = Transaction.fromBEEF(fixture.certificateBEEF) + badTransaction.merklePath = new MerklePath(700_000, [ + [ + { offset: 0, hash: badTransaction.id('hex'), txid: true }, + { offset: 1, hash: '42'.repeat(32) } + ] + ]) + const badReceipt = badTransaction.toBEEF() + expect(Transaction.fromBEEF(badReceipt).id('hex')).toBe(fixture.certificateTransaction.id('hex')) + const firstHost = 'https://first.invalid-proof.example' + const secondHost = 'https://second.valid-proof.example' + const lookup = jest.fn(async (host: string) => + host === firstHost + ? { type: 'output-list' as const, outputs: [{ beef: badReceipt, outputIndex: 0 }] } + : { type: 'output-list' as const, outputs: [{ beef: fixture.certificateBEEF, outputIndex: 0 }] } + ) + const resolver = new LookupResolver({ + facilitator: { lookup }, + hostOverrides: { ls_identity: [firstHost, secondHost] } + }) + const { wallet, trustSettings } = walletFor(fixture, resolver) + const certificateVerify = jest.spyOn(VerifiableCertificate.prototype, 'verify') + const args = { identityKey: fixture.certificate.subject } + + const initial = await wallet.discoverByIdentityKey(args) + expect(initial).toMatchObject({ totalCertificates: 1 }) + expect(lookup).toHaveBeenCalledTimes(2) + expect(certificateVerify).toHaveBeenCalledTimes(1) + + initial.certificates[0].decryptedFields.name = 'Mutated caller copy' + trustSettings.trustLevel = 2 + await expect(wallet.discoverByIdentityKey(args)).resolves.toEqual({ totalCertificates: 0, certificates: [] }) + expect(certificateVerify).toHaveBeenCalledTimes(1) + + trustSettings.trustLevel = 1 + const restored = await wallet.discoverByIdentityKey(args) + expect(restored.certificates[0].decryptedFields).toEqual({ name: 'Alice' }) + // The rejected receipt makes the bounded raw-receipt cache retry, but the + // verified certificate is still reused across trust-only refiltering. + expect(lookup).toHaveBeenCalledTimes(6) + expect(certificateVerify).toHaveBeenCalledTimes(1) + certificateVerify.mockRestore() + }) + it('keeps trust filtering and forceRefresh on the final Promise API', async () => { const { wallet, query, trustSettings } = walletFor(fixture) const args = { identityKey: fixture.certificate.subject } @@ -161,13 +215,21 @@ describe('Wallet final identity verification and compatibility', () => { query: { identityKey: fixture.certificate.subject, certifiers: [fixture.certificate.certifier] } }, undefined, - { graceMs: 300 } + { + graceMs: 300, + evidenceLimits: { maxOutputs: 512, maxBytes: 16 * 1024 * 1024 }, + onEvidence: expect.any(Function) + } ) await wallet.discoverByAttributes({ attributes: { name: 'Alice' }, limit: 1, offset: 3, seekPermission: true }) expect(query).toHaveBeenLastCalledWith( { service: 'ls_identity', query: { attributes: { name: 'Alice' }, certifiers: [fixture.certificate.certifier] } }, undefined, - { graceMs: 300 } + { + graceMs: 300, + evidenceLimits: { maxOutputs: 512, maxBytes: 16 * 1024 * 1024 }, + onEvidence: expect.any(Function) + } ) await expect(wallet.discoverByIdentityKey({ ...identityArgs, limit: 10_001 })).rejects.toThrow() await expect(wallet.discoverByAttributes({ attributes: {}, offset: -1 })).rejects.toThrow() diff --git a/packages/wallet/wallet-toolbox/src/services/Services.ts b/packages/wallet/wallet-toolbox/src/services/Services.ts index 50cb4d8e4..677e6409c 100644 --- a/packages/wallet/wallet-toolbox/src/services/Services.ts +++ b/packages/wallet/wallet-toolbox/src/services/Services.ts @@ -48,6 +48,8 @@ import { asArray, asString } from '../utility/utilityHelpers.noBuffer' import { classifyOutputUtxo, requireConclusiveUtxo } from './classifyOutputUtxo' export class Services implements WalletServices { + private identityChainTracker?: ChaintracksChainTracker + private identityChainTrackerInit?: Promise static readonly getStatusForTxidsBatchLimit = 20 static createDefaultOptions(chain: Chain): WalletServicesOptions { @@ -229,16 +231,36 @@ export class Services implements WalletServices { } async getChainTracker(): Promise { - if (this.options.chainTracker != null) return this.options.chainTracker - if (this.options.chaintracks == null) { - throw new WERR_INVALID_PARAMETER( - 'options.chainTracker or options.chaintracks', - "valid to enable 'getChainTracker' service." - ) + while (true) { + if (this.options.chainTracker != null) return this.options.chainTracker + if (this.options.chaintracks == null) { + throw new WERR_INVALID_PARAMETER( + 'options.chainTracker or options.chaintracks', + "valid to enable 'getChainTracker' service." + ) + } + const desired = this.options.chaintracks + if (this.identityChainTracker?.chaintracks === desired) return this.identityChainTracker + if (this.identityChainTrackerInit != null) { + await this.identityChainTrackerInit.catch(() => undefined) + continue + } + const previous = this.identityChainTracker + const created = new ChaintracksChainTracker(this.chain, desired, { + telemetry: this.options.telemetry + }) + this.identityChainTracker = created + const init = Promise.resolve().then(async () => { + if (previous != null) await previous.dispose() + return created + }) + this.identityChainTrackerInit = init + try { + await init.catch(() => created) + } finally { + if (this.identityChainTrackerInit === init) this.identityChainTrackerInit = undefined + } } - return new ChaintracksChainTracker(this.chain, this.options.chaintracks, { - telemetry: this.options.telemetry - }) } async getBsvExchangeRate(): Promise { diff --git a/packages/wallet/wallet-toolbox/src/services/__tests/verifyBeef.test.ts b/packages/wallet/wallet-toolbox/src/services/__tests/verifyBeef.test.ts index 58238d326..d8576e43e 100644 --- a/packages/wallet/wallet-toolbox/src/services/__tests/verifyBeef.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/__tests/verifyBeef.test.ts @@ -64,6 +64,35 @@ describe('verifyBeef tests', () => { await expect(services.getChainTracker()).resolves.toBe(chainTracker) }) + test('coalesces concurrent getChainTracker lookups onto one wrapper', async () => { + const services = new Services('main') + const [first, second] = await Promise.all([services.getChainTracker(), services.getChainTracker()]) + expect(first).toBe(second) + await expect(services.getChainTracker()).resolves.toBe(first) + }) + + test('publishes one replacement before disposing a mismatched tracker', async () => { + const firstClient = { + getPresentHeight: jest.fn(async () => 1), + findHeaderForHeight: jest.fn(), + findChainTipHash: jest.fn(async () => 'aa'.repeat(32)) + } + const secondClient = { + getPresentHeight: jest.fn(async () => 2), + findHeaderForHeight: jest.fn(), + findChainTipHash: jest.fn(async () => 'bb'.repeat(32)) + } + const options = { ...Services.createDefaultOptions('main'), chaintracks: firstClient as any } + const services = new Services(options) + const original = await services.getChainTracker() + + services.options.chaintracks = secondClient as any + const [first, second] = await Promise.all([services.getChainTracker(), services.getChainTracker()]) + expect(first).toBe(second) + expect(first).not.toBe(original) + await expect(services.getChainTracker()).resolves.toBe(first) + }) + test('1_', async () => { if (_tu.noEnv('main')) return const { env: _env, storage, services } = await _tu.createMainReviewSetup() diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts index 9ca723e86..b5bec6a09 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts @@ -24,6 +24,8 @@ interface BHSHeaderState { } export class BHServiceClient implements ChaintracksServiceClient { + /** HTTP polling client; callback event methods are legacy unsupported stubs. */ + readonly supportsReorgEvents = false bhs: BlockHeadersService cache: Record chain: Chain diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/ChaintracksChainTracker.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/ChaintracksChainTracker.ts index d953c7d06..0a90d1ad8 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/ChaintracksChainTracker.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/ChaintracksChainTracker.ts @@ -13,56 +13,174 @@ export interface ChaintracksChainTrackerOptions { } export class ChaintracksChainTracker implements ChainTracker { - chaintracks: ChaintracksClientApi + private _chaintracks: ChaintracksClientApi + private verificationContext = 0 + private reorgEpoch = 0 + private observerGeneration = 0 + private reorgSubscription?: string + private reorgProvider?: ChaintracksClientApi + private reorgSetup?: Promise + private readonly cacheUpdatedAt: Record = {} cache: Record options: ChaintracksChainTrackerOptions readonly telemetry: Telemetry constructor(chain?: Chain, chaintracks?: ChaintracksClientApi, options?: ChaintracksChainTrackerOptions) { chain ||= 'main' - this.chaintracks = + this._chaintracks = chaintracks ?? new ChaintracksServiceClient(chain, `https://${chain}net-chaintracks.babbage.systems`) this.cache = {} this.options = options || {} this.telemetry = new Telemetry(this.options.telemetry) } - async currentHeight(): Promise { - if (!this.telemetry.enabled) return await this.chaintracks.getPresentHeight() + get chaintracks(): ChaintracksClientApi { + return this._chaintracks + } + + set chaintracks(value: ChaintracksClientApi) { + if (value === this._chaintracks) return + this._chaintracks = value + this.verificationContext++ + this.observerGeneration++ + this.cache = {} + for (const height of Object.keys(this.cacheUpdatedAt)) delete this.cacheUpdatedAt[Number(height)] + void this.releaseReorgListener() + } + + /** Local provider-generation marker for consumers that invalidate derived verdicts. */ + getVerificationContext(): string { + return `chaintracks:${this.verificationContext}:${this.reorgEpoch}:${this.observerGeneration}` + } + + /** Fresh provider-bound canonical tip token; unlike the stable context, normal tip advances change it. */ + async getVerificationContextToken(signal?: AbortSignal): Promise { + this.throwIfAborted(signal) + const chaintracks = this.chaintracks + const context = this.verificationContext + const marker = this.getVerificationContext() + await this.ensureReorgListener(chaintracks, context) + this.throwIfAborted(signal) + if (this.getVerificationContext() !== marker) + throw new Error('Chaintracks provider changed during event registration') + const tip = await chaintracks.findChainTipHash() + this.throwIfAborted(signal) + if ( + this.chaintracks !== chaintracks || + this.verificationContext !== context || + this.getVerificationContext() !== marker + ) { + throw new Error('Chaintracks provider changed during canonical token lookup') + } + return JSON.stringify([marker, tip]) + } + + async dispose(): Promise { + this.observerGeneration++ + await this.releaseReorgListener() + } + + private reorgSubscribe(provider: ChaintracksClientApi): ((listener: unknown) => Promise) | undefined { + if (provider.supportsReorgEvents === false) return undefined + const events = provider as ChaintracksClientApi & { + subscribeReorgs?: (listener: unknown) => Promise + } + if (typeof events.subscribeReorgs === 'function') return async listener => await events.subscribeReorgs!(listener) + if (provider.supportsReorgEvents === true) { + throw new Error('Chaintracks provider promised reorg events but subscribeReorgs is not implemented') + } + return undefined + } + + private async ensureReorgListener(provider: ChaintracksClientApi, context: number): Promise { + const subscribe = this.reorgSubscribe(provider) + if (subscribe == null) return + if (this.reorgProvider === provider && this.reorgSubscription != null) return + if (this.reorgSetup != null) { + await this.reorgSetup + if (this.chaintracks !== provider || this.verificationContext !== context) + throw new Error('Chaintracks provider changed during event registration') + if (this.reorgProvider === provider && this.reorgSubscription != null) return + } + const generation = this.observerGeneration + this.reorgSetup = (async () => { + const subscription = await subscribe(() => { + if ( + this.chaintracks === provider && + this.verificationContext === context && + this.observerGeneration === generation + ) + this.reorgEpoch++ + }) + if ( + this.chaintracks !== provider || + this.verificationContext !== context || + this.observerGeneration !== generation + ) { + await provider.unsubscribe(subscription).catch(() => undefined) + return + } + this.reorgProvider = provider + this.reorgSubscription = subscription + })() + try { + await this.reorgSetup + } finally { + this.reorgSetup = undefined + } + } + + private async releaseReorgListener(): Promise { + const provider = this.reorgProvider + const subscription = this.reorgSubscription + this.reorgProvider = undefined + this.reorgSubscription = undefined + if (provider != null && subscription != null) await provider.unsubscribe(subscription).catch(() => undefined) + } + + async currentHeight(signal?: AbortSignal): Promise { + this.throwIfAborted(signal) + const chaintracks = this.chaintracks + const context = this.verificationContext + const currentHeight = async (): Promise => { + const height = await chaintracks.getPresentHeight() + this.throwIfAborted(signal) + if (this.chaintracks !== chaintracks || this.verificationContext !== context) + throw new Error('Chaintracks provider changed during height lookup') + return height + } + if (!this.telemetry.enabled) return await currentHeight() return await this.telemetry.withSpan( 'wallet.chaintracks.current_height', { component: 'chaintracks-chain-tracker', kind: 'client' }, - async () => await this.chaintracks.getPresentHeight() + async () => await currentHeight() ) } - async isValidRootForHeight(root: string, height: number): Promise { - if (!this.telemetry.enabled) return await this.isValidRootForHeightCore(root, height) + async isValidRootForHeight(root: string, height: number, signal?: AbortSignal): Promise { + this.throwIfAborted(signal) + if (!this.telemetry.enabled) return await this.isValidRootForHeightCore(root, height, undefined, signal) return await this.telemetry.withSpan( 'wallet.chaintracks.validate_root', { component: 'chaintracks-chain-tracker', kind: 'client' }, - async span => await this.isValidRootForHeightCore(root, height, span) + async span => await this.isValidRootForHeightCore(root, height, span, signal) ) } - private async isValidRootForHeightCore(root: string, height: number, parent?: TelemetrySpan): Promise { - const cachedRoot = this.cache[height] - if (cachedRoot) { - parent?.end({ - attributes: { - 'chaintracks.cache_hit': true, - 'chaintracks.valid': cachedRoot === root - } - }) - return cachedRoot === root - } - + private async isValidRootForHeightCore( + root: string, + height: number, + parent?: TelemetrySpan, + signal?: AbortSignal + ): Promise { + const chaintracks = this.chaintracks + const context = this.verificationContext let header: BlockHeader | undefined const retries = Math.max(1, this.options.maxRetries ?? 6) @@ -72,9 +190,13 @@ export class ChaintracksChainTracker implements ChainTracker { for (let tryCount = 1; tryCount <= retries; tryCount++) { try { + this.throwIfAborted(signal) + if (this.chaintracks !== chaintracks || this.verificationContext !== context) { + throw new Error('Chaintracks provider changed during header lookup') + } header = parent == null - ? await this.chaintracks.findHeaderForHeight(height) + ? await chaintracks.findHeaderForHeight(height) : await this.telemetry.withSpan( 'wallet.chaintracks.find_header', { @@ -85,28 +207,47 @@ export class ChaintracksChainTracker implements ChainTracker { 'retry.attempt': tryCount } }, - async () => await this.chaintracks.findHeaderForHeight(height) + async () => await chaintracks.findHeaderForHeight(height) ) + if (this.chaintracks !== chaintracks || this.verificationContext !== context) { + throw new Error('Chaintracks provider changed during header lookup') + } + if (header == null) { if (tryCount >= retries) return false + this.throwIfAborted(signal) await wait(retryDelayMs) continue } break } catch (error_: unknown) { + this.throwIfAborted(signal) + if (this.chaintracks !== chaintracks || this.verificationContext !== context) { + throw new Error('Chaintracks provider changed during header lookup') + } error = WalletError.fromUnknown(error_) if (tryCount >= retries) { throw error } + this.throwIfAborted(signal) await wait(retryDelayMs) } } if (header == null) return false + // Diagnostic only: a root is always freshly read from the current canonical source above. this.cache[height] = header.merkleRoot + this.cacheUpdatedAt[height] = Date.now() + const now = Date.now() + const entries = Object.entries(this.cacheUpdatedAt).sort(([, a], [, b]) => a - b) + for (const [index, [cachedHeight, updatedAt]] of entries.entries()) { + if (index >= entries.length - 256 && now - updatedAt <= 5 * 60 * 1000) continue + delete this.cache[Number(cachedHeight)] + delete this.cacheUpdatedAt[Number(cachedHeight)] + } const valid = header.merkleRoot === root parent?.end({ @@ -117,4 +258,8 @@ export class ChaintracksChainTracker implements ChainTracker { }) return valid } + + private throwIfAborted(signal?: AbortSignal): void { + if (signal?.aborted === true) throw signal.reason ?? new Error('Chaintracks verification aborted') + } } diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/LocalChainTracker.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/LocalChainTracker.ts index 900c8e28e..9267eb758 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/LocalChainTracker.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/LocalChainTracker.ts @@ -80,6 +80,12 @@ export class LocalChainTracker implements ChainTracker { private readonly recoverLocal: LocalChainTrackerOptions['recoverLocal'] private readonly clearLocal: LocalChainTrackerOptions['clearLocal'] private readonly now: () => Date + private verificationContext = 0 + private resetting = false + private eventEpoch = 0 + private observerGeneration = 0 + private readonly eventSubscriptions = new Map() + private readonly eventSetups = new Map>() private status: LocalChainTrackerStatus constructor(options: LocalChainTrackerOptions) { @@ -111,6 +117,7 @@ export class LocalChainTracker implements ChainTracker { } setMode(mode: LocalChainTrackerMode): void { + if (this.status.mode !== mode) this.verificationContext++ this.status = { ...this.status, mode, @@ -118,6 +125,46 @@ export class LocalChainTracker implements ChainTracker { } } + /** Local mode/source generation marker for consumers that invalidate derived verdicts. */ + getVerificationContext(): string { + return `local-chaintracks:${this.sourceContext()}` + } + + /** + * Fresh token for providers participating in this attempt's canonical authority. + * Remote-only uses fallbacks only; local-primary uses the local client only. + * A missing participating identity fails closed. Unused providers are omitted + * and cannot stand in. This is not an atomic multi-source snapshot. + */ + async getVerificationContextToken(signal?: AbortSignal): Promise { + const context = this.sourceContext() + this.throwIfAborted(signal) + this.assertCurrentContext(context, 'canonical token lookup') + const sources = this.attemptSources() + if (sources.length === 0) throw new Error('No canonical ChainTracks source is available') + await this.ensureEventObservers(sources) + const tips = await Promise.all(sources.map(async source => await source.findChainTipHash())) + this.throwIfAborted(signal) + this.assertCurrentContext(context, 'canonical token lookup') + if (!this.sameSources(sources, this.attemptSources())) { + throw new Error('Local ChainTracks provider changed during canonical token lookup') + } + if (tips.some(tip => tip == null || tip === '')) { + throw new Error('Canonical ChainTracks source identity is unavailable') + } + return JSON.stringify({ context, eventEpoch: this.eventEpoch, tips }) + } + + async dispose(): Promise { + this.observerGeneration++ + await Promise.all( + Array.from(this.eventSubscriptions.entries()).map(async ([source, subscription]) => { + this.eventSubscriptions.delete(source) + await source.unsubscribe(subscription).catch(() => undefined) + }) + ) + } + getStatus(): LocalChainTrackerStatus { return { ...this.status } } @@ -127,32 +174,48 @@ export class LocalChainTracker implements ChainTracker { } async currentHeight(): Promise { + const context = this.sourceContext() + this.assertCurrentContext(context, 'height lookup') if (this.status.mode === 'remote-only') { - return await this.fallbackHeight() + const height = await this.fallbackHeight() + this.assertCurrentContext(context, 'height lookup') + return height } + const local = this.local try { - const height = await this.local.getPresentHeight() + const height = await local.getPresentHeight() + this.assertCurrentContext(context, 'height lookup') this.status = { ...this.status, activeSource: 'local', localHeight: height, lastError: undefined } return height } catch (error) { + this.assertCurrentContext(context, 'height lookup') this.recordError(error) if (!this.fallbackOnLocalError) throw error - return await this.fallbackHeight() + const height = await this.fallbackHeight() + this.assertCurrentContext(context, 'height lookup') + return height } } async isValidRootForHeight(root: string, height: number): Promise { + const context = this.sourceContext() + this.assertCurrentContext(context, 'root validation') if (this.status.mode !== 'remote-only') { + const local = this.local try { - const valid = await this.local.isValidRootForHeight(root, height) + const valid = await local.isValidRootForHeight(root, height) + this.assertCurrentContext(context, 'root validation') this.status = { ...this.status, activeSource: 'local', lastError: undefined } return valid } catch (error) { + this.assertCurrentContext(context, 'root validation') this.recordError(error) if (!this.fallbackOnLocalError) throw error } } - return await this.fallbackValidation(root, height) + const valid = await this.fallbackValidation(root, height) + this.assertCurrentContext(context, 'root validation') + return valid } async synchronize(): Promise { @@ -162,8 +225,10 @@ export class LocalChainTracker implements ChainTracker { } async clearLocalData(): Promise { - if (this.clearLocal == null) throw new Error('Local ChainTracks clearing is not configured.') - this.local = await this.clearLocal() + const clearLocal = this.clearLocal + if (clearLocal == null) throw new Error('Local ChainTracks clearing is not configured.') + const reset = this.beginReset() + await this.applyReset(reset, clearLocal) this.status = { mode: this.status.mode, activeSource: this.status.mode === 'remote-only' ? 'fallback' : 'local', @@ -274,15 +339,20 @@ export class LocalChainTracker implements ChainTracker { this.recoverLocal != null && expectedHash != null ) { - this.local = await this.recoverLocal({ - reason: consistency, - localHeight, - referenceHeight, - heightLag, - comparisonHeight, - expectedHash, - referenceAgreement - }) + const reset = this.beginReset() + await this.applyReset( + reset, + async () => + await this.recoverLocal!({ + reason: consistency, + localHeight, + referenceHeight, + heightLag, + comparisonHeight, + expectedHash, + referenceAgreement + }) + ) const recoveredAt = this.now().toISOString() const recovered = await this.checkConsistencyInternal(false) this.status = { ...recovered, recoveredAt } @@ -315,6 +385,114 @@ export class LocalChainTracker implements ChainTracker { throw error } + private attemptSources(): ChaintracksClientApi[] { + return this.status.mode === 'remote-only' ? [...this.fallbacks] : [this.local] + } + + private sameSources(left: ChaintracksClientApi[], right: ChaintracksClientApi[]): boolean { + return left.length === right.length && left.every((source, index) => source === right[index]) + } + + private async applyReset(reset: number, nextLocal: () => Promise): Promise { + await this.dispose() + this.assertResetOwner(reset) + const local = await nextLocal() + this.assertResetOwner(reset) + this.local = local + this.resetting = false + } + + private sourceContext(): string { + const nested = (source: ChaintracksClientApi): string => { + const provider = source as ChaintracksClientApi & { getVerificationContext?: () => string | number } + return String(provider.getVerificationContext?.() ?? '') + } + return JSON.stringify([ + this.verificationContext, + this.resetting, + this.status.mode, + this.eventEpoch, + this.observerGeneration, + nested(this.local), + this.fallbacks.map(nested) + ]) + } + + private assertCurrentContext(context: string, operation: string): void { + if (this.resetting || context !== this.sourceContext()) { + throw new Error(`Local ChainTracks provider changed during ${operation}`) + } + } + + private throwIfAborted(signal?: AbortSignal): void { + if (signal?.aborted === true) throw signal.reason ?? new Error('Local ChainTracks token lookup aborted') + } + + private beginReset(): number { + this.verificationContext++ + this.resetting = true + return this.verificationContext + } + + private assertResetOwner(reset: number): void { + if (reset !== this.verificationContext) throw new Error('Local ChainTracks reset was superseded') + } + + private reorgSubscribe(source: ChaintracksClientApi): ((listener: unknown) => Promise) | undefined { + if (source.supportsReorgEvents === false) return undefined + const events = source as ChaintracksClientApi & { + subscribeReorgs?: (listener: unknown) => Promise + } + if (typeof events.subscribeReorgs === 'function') return async listener => await events.subscribeReorgs!(listener) + if (source.supportsReorgEvents === true) { + throw new Error('Chaintracks provider promised reorg events but subscribeReorgs is not implemented') + } + return undefined + } + + private async ensureEventObservers(sources: ChaintracksClientApi[]): Promise { + await Promise.all( + sources.map(async source => { + const subscribe = this.reorgSubscribe(source) + if (subscribe == null) return + if (this.eventSubscriptions.has(source)) return + const pending = this.eventSetups.get(source) + if (pending !== undefined) { + await pending + if ( + !this.eventSubscriptions.has(source) && + !this.resetting && + (source === this.local || this.fallbacks.includes(source)) + ) { + await this.ensureEventObservers([source]) + } + return + } + const generation = this.observerGeneration + const current = (): boolean => + !this.resetting && + generation === this.observerGeneration && + (source === this.local || this.fallbacks.includes(source)) + const setup = (async () => { + const subscription = await subscribe(() => { + if (current()) this.eventEpoch++ + }) + if (!current()) { + await source.unsubscribe(subscription).catch(() => undefined) + return + } + this.eventSubscriptions.set(source, subscription) + })() + this.eventSetups.set(source, setup) + try { + await setup + } finally { + if (this.eventSetups.get(source) === setup) this.eventSetups.delete(source) + } + }) + ) + } + private async fallbackValidation(root: string, height: number): Promise { let valid = 0 let invalid = 0 diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts index 3a7cbfaef..32d23a42e 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts @@ -1,4 +1,5 @@ import { ChaintracksChainTracker } from '../index.all' +import { ChaintracksServiceClient } from '../chaintracks/ChaintracksServiceClient' import { sdk } from '../../../index.client' import { BlockHeader } from '../../../sdk/WalletServices.interfaces' @@ -71,7 +72,7 @@ describe('ChaintracksChaintracker tests', () => { test('traces retry attempts and cache disposition without roots or headers', async () => { const events: any[] = [] let nextSpanId = 1 - const chaintracks = makeChaintracksClient([undefined, HEADER_877599]) + const chaintracks = makeChaintracksClient([undefined, HEADER_877599, HEADER_877599]) const tracker = new ChaintracksChainTracker('main', chaintracks, { maxRetries: 2, retryDelayMs: 0, @@ -90,15 +91,15 @@ describe('ChaintracksChaintracker tests', () => { const attempts = events.filter(event => event.name === 'wallet.chaintracks.find_header') const validations = events.filter(event => event.name === 'wallet.chaintracks.validate_root') - expect(attempts).toHaveLength(2) - expect(attempts.map(event => event.attributes['retry.attempt'])).toEqual([1, 2]) + expect(attempts).toHaveLength(3) + expect(attempts.map(event => event.attributes['retry.attempt'])).toEqual([1, 2, 1]) expect(validations).toHaveLength(2) expect(validations[0].attributes).toMatchObject({ 'chaintracks.cache_hit': false, 'chaintracks.valid': true }) expect(validations[1].attributes).toMatchObject({ - 'chaintracks.cache_hit': true, + 'chaintracks.cache_hit': false, 'chaintracks.valid': true }) expect(events.find(event => event.name === 'wallet.chaintracks.current_height')).toMatchObject({ @@ -107,6 +108,127 @@ describe('ChaintracksChaintracker tests', () => { expect(JSON.stringify(events)).not.toContain(HEADER_877599.merkleRoot) expect(JSON.stringify(events)).not.toContain(HEADER_877599.previousHash) }) + + test('reads the current canonical root on every request after a same-height reorg', async () => { + const reorged = { ...HEADER_877599, merkleRoot: '11'.repeat(32) } + const chaintracks = makeChaintracksClient([HEADER_877599, reorged]) + const tracker = new ChaintracksChainTracker('main', chaintracks, { maxRetries: 1 }) + + await expect(tracker.isValidRootForHeight(HEADER_877599.merkleRoot, HEADER_877599.height)).resolves.toBe(true) + await expect(tracker.isValidRootForHeight(HEADER_877599.merkleRoot, HEADER_877599.height)).resolves.toBe(false) + expect(chaintracks.findHeaderForHeight).toHaveBeenCalledTimes(2) + }) + + test('rejects a header result from a provider replaced while its request is pending', async () => { + let release: (() => void) | undefined + const pending = new Promise(resolve => { + release = resolve + }) + const oldProvider = makeChaintracksClient([]) + oldProvider.findHeaderForHeight.mockImplementation(async () => { + await pending + return HEADER_877599 + }) + const replacement = makeChaintracksClient([HEADER_877599]) + const tracker = new ChaintracksChainTracker('main', oldProvider, { maxRetries: 3, retryDelayMs: 0 }) + + const request = tracker.isValidRootForHeight(HEADER_877599.merkleRoot, HEADER_877599.height) + tracker.chaintracks = replacement + tracker.chaintracks = oldProvider + release!() + + await expect(request).rejects.toThrow('provider changed') + expect(oldProvider.findHeaderForHeight).toHaveBeenCalledTimes(1) + expect(tracker.getVerificationContext()).toContain('chaintracks:2:') + expect(tracker.cache).toEqual({}) + }) + + test('changes a token for every reorg event, including an ABA tip, and releases its listener', async () => { + let listener: (() => void) | undefined + const provider = makeChaintracksClient([]) + provider.findChainTipHash = jest.fn(async () => 'aa'.repeat(32)) + provider.subscribeReorgs = jest.fn(async callback => { + listener = callback + return 'reorg-1' + }) + provider.unsubscribe = jest.fn(async () => true) + const tracker = new ChaintracksChainTracker('main', provider) + + const before = await tracker.getVerificationContextToken() + listener!() + const after = await tracker.getVerificationContextToken() + expect(after).not.toBe(before) + await tracker.dispose() + expect(provider.unsubscribe).toHaveBeenCalledWith('reorg-1') + }) + + test('unsubscribes a deferred registration after provider replacement and obtains a fresh subscription', async () => { + let release: ((value: string) => void) | undefined + const pending = new Promise(resolve => { + release = resolve + }) + const oldProvider = makeChaintracksClient([]) + oldProvider.findChainTipHash = jest.fn(async () => 'aa'.repeat(32)) + oldProvider.subscribeReorgs = jest.fn(async () => await pending) + oldProvider.unsubscribe = jest.fn(async () => true) + const replacement = makeChaintracksClient([]) + replacement.findChainTipHash = jest.fn(async () => 'bb'.repeat(32)) + replacement.subscribeReorgs = jest.fn(async () => 'fresh-subscription') + replacement.unsubscribe = jest.fn(async () => true) + const tracker = new ChaintracksChainTracker('main', oldProvider) + + const staleToken = tracker.getVerificationContextToken() + tracker.chaintracks = replacement + release!('stale-subscription') + await expect(staleToken).rejects.toThrow('provider changed') + expect(oldProvider.unsubscribe).toHaveBeenCalledWith('stale-subscription') + await expect(tracker.getVerificationContextToken()).resolves.toContain('bb'.repeat(32)) + expect(replacement.subscribeReorgs).toHaveBeenCalledTimes(1) + }) + + test('rejects a token when dispose races with pending registration on the same provider', async () => { + let release: ((value: string) => void) | undefined + const pending = new Promise(resolve => { + release = resolve + }) + const provider = makeChaintracksClient([]) + provider.findChainTipHash = jest.fn(async () => 'aa'.repeat(32)) + provider.subscribeReorgs = jest.fn(async () => await pending) + provider.unsubscribe = jest.fn(async () => true) + const tracker = new ChaintracksChainTracker('main', provider) + + const token = tracker.getVerificationContextToken() + const disposed = tracker.dispose() + release!('disposed-pending') + await disposed + await expect(token).rejects.toThrow('provider changed') + expect(provider.unsubscribe).toHaveBeenCalledWith('disposed-pending') + }) + + test('uses the HTTP Chaintracks client without attempting its unsupported reorg subscription', async () => { + const provider = new ChaintracksServiceClient('main', 'https://chaintracks.example') + expect(provider.supportsReorgEvents).toBe(false) + jest.spyOn(provider, 'findChainTipHash').mockResolvedValue('aa'.repeat(32)) + jest.spyOn(provider, 'subscribeReorgs') + const tracker = new ChaintracksChainTracker('main', provider) + + await expect(tracker.getVerificationContextToken()).resolves.toContain('aa'.repeat(32)) + expect(provider.subscribeReorgs).not.toHaveBeenCalled() + }) + + test('does not hide a registration failure from a built-in client that promises reorg events', async () => { + class PromisingEventsClient extends ChaintracksServiceClient { + override readonly supportsReorgEvents = true + } + const provider = new PromisingEventsClient('main', 'https://chaintracks.example') + expect(provider.supportsReorgEvents).toBe(true) + jest.spyOn(provider, 'findChainTipHash').mockResolvedValue('aa'.repeat(32)) + const subscribe = jest.spyOn(provider, 'subscribeReorgs') + const tracker = new ChaintracksChainTracker('main', provider) + + await expect(tracker.getVerificationContextToken()).rejects.toThrow('Method not implemented.') + expect(subscribe).toHaveBeenCalled() + }) }) async function testChaintracksChaintracker(chain: sdk.Chain) { diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/LocalChainTracker.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/LocalChainTracker.test.ts index 913fc4cb4..feef8f630 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/LocalChainTracker.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/LocalChainTracker.test.ts @@ -1,4 +1,5 @@ import type { ChaintracksClientApi } from '../chaintracks/Api/ChaintracksClientApi' +import { ChaintracksServiceClient } from '../chaintracks/ChaintracksServiceClient' import { LocalChainTracker } from '../LocalChainTracker' const localHash = '01'.repeat(32) @@ -40,7 +41,65 @@ function client( } as unknown as ChaintracksClientApi } +class PromisingEventsClient extends ChaintracksServiceClient { + override readonly supportsReorgEvents = true +} + describe('LocalChainTracker', () => { + test('uses an event-unsupported HTTP fallback in remote-only mode without subscribing', async () => { + const local = client() + const fallback = new ChaintracksServiceClient('main', 'https://chaintracks.example') + expect(fallback.supportsReorgEvents).toBe(false) + jest.spyOn(fallback, 'findChainTipHash').mockResolvedValue('aa'.repeat(32)) + jest.spyOn(fallback, 'subscribeReorgs') + const tracker = new LocalChainTracker({ local, fallbacks: [fallback], mode: 'remote-only' }) + + await expect(tracker.getVerificationContextToken()).resolves.toContain('aa'.repeat(32)) + expect(fallback.subscribeReorgs).not.toHaveBeenCalled() + expect(local.findChainTipHash).not.toHaveBeenCalled() + }) + + test('does not hide a registration failure from a fallback that promises reorg events', async () => { + const fallback = new PromisingEventsClient('main', 'https://chaintracks.example') + expect(fallback.supportsReorgEvents).toBe(true) + jest.spyOn(fallback, 'findChainTipHash').mockResolvedValue('aa'.repeat(32)) + const subscribe = jest.spyOn(fallback, 'subscribeReorgs') + const tracker = new LocalChainTracker({ local: client(), fallbacks: [fallback], mode: 'remote-only' }) + + await expect(tracker.getVerificationContextToken()).rejects.toThrow('Method not implemented.') + expect(subscribe).toHaveBeenCalled() + }) + + test('requires participating canonical identity and ignores unused providers', async () => { + const unusedLocal = client({ hash: 'aa'.repeat(32) }) + const deadFallback = client({ tipError: new Error('fallback offline') }) + const remoteOnly = new LocalChainTracker({ + local: unusedLocal, + fallbacks: [deadFallback], + mode: 'remote-only' + }) + await expect(remoteOnly.getVerificationContextToken()).rejects.toThrow('fallback offline') + expect(unusedLocal.findChainTipHash).not.toHaveBeenCalled() + + const hangingLocal = client({ hash: 'aa'.repeat(32) }) + const remoteWithoutFallback = new LocalChainTracker({ local: hangingLocal, mode: 'remote-only' }) + await expect(remoteWithoutFallback.getVerificationContextToken()).rejects.toThrow( + 'No canonical ChainTracks source is available' + ) + expect(hangingLocal.findChainTipHash).not.toHaveBeenCalled() + + const liveLocal = client({ hash: 'cc'.repeat(32) }) + const unusedFallback = client({ tipError: new Error('fallback offline') }) + const localPrimary = new LocalChainTracker({ local: liveLocal, fallbacks: [unusedFallback] }) + await expect(localPrimary.getVerificationContextToken()).resolves.toContain('cc'.repeat(32)) + expect(unusedFallback.findChainTipHash).not.toHaveBeenCalled() + + const deadLocal = client({ tipError: new Error('local offline') }) + const unusedLiveFallback = client({ hash: 'dd'.repeat(32) }) + const localMissing = new LocalChainTracker({ local: deadLocal, fallbacks: [unusedLiveFallback] }) + await expect(localMissing.getVerificationContextToken()).rejects.toThrow('local offline') + expect(unusedLiveFallback.findChainTipHash).not.toHaveBeenCalled() + }) test('never overrides a definitive local rejection with a remote answer', async () => { const local = client({ valid: false }) const fallback = client({ valid: true }) @@ -274,6 +333,172 @@ describe('LocalChainTracker', () => { expect(local.listening).toHaveBeenCalledTimes(1) }) + test('fences an in-flight root validation while local clearing is deferred or fails', async () => { + let releaseRoot: (() => void) | undefined + let releaseClear: (() => void) | undefined + const rootPending = new Promise(resolve => { + releaseRoot = resolve + }) + const clearPending = new Promise(resolve => { + releaseClear = () => resolve(client()) + }) + const local = client() + ;(local.isValidRootForHeight as jest.Mock).mockImplementation(async () => { + await rootPending + return true + }) + const tracker = new LocalChainTracker({ local, clearLocal: async () => await clearPending }) + const validation = tracker.isValidRootForHeight('root', 100) + const clearing = tracker.clearLocalData() + releaseRoot!() + await expect(validation).rejects.toThrow('provider changed') + releaseClear!() + await expect(clearing).resolves.toMatchObject({ consistency: 'unchecked' }) + + const failed = new LocalChainTracker({ + local: client(), + clearLocal: async () => { + throw new Error('reset failed') + } + }) + await expect(failed.clearLocalData()).rejects.toThrow('reset failed') + await expect(failed.isValidRootForHeight('root', 100)).rejects.toThrow('provider changed') + }) + + test('drops a pending observer registration during reset and registers a fresh replacement observer', async () => { + let releaseSubscription: ((value: string) => void) | undefined + const pendingSubscription = new Promise(resolve => { + releaseSubscription = resolve + }) + const oldLocal = client() + ;(oldLocal as any).subscribeReorgs = jest.fn(async () => await pendingSubscription) + ;(oldLocal as any).unsubscribe = jest.fn(async () => true) + const replacement = client() + ;(replacement as any).subscribeReorgs = jest.fn(async () => 'fresh') + ;(replacement as any).unsubscribe = jest.fn(async () => true) + const tracker = new LocalChainTracker({ local: oldLocal, clearLocal: async () => replacement }) + + const staleToken = tracker.getVerificationContextToken() + const reset = tracker.clearLocalData() + releaseSubscription!('stale') + await expect(staleToken).rejects.toThrow('provider changed') + await reset + await expect(tracker.getVerificationContextToken()).resolves.toContain('eventEpoch') + expect((oldLocal as any).unsubscribe).toHaveBeenCalledWith('stale') + expect((replacement as any).subscribeReorgs).toHaveBeenCalledTimes(1) + }) + + test('does not invoke a superseded recovery hook when reset is overtaken during disposal', async () => { + let releaseUnsubscribe: (() => void) | undefined + const pendingUnsubscribe = new Promise(resolve => { + releaseUnsubscribe = resolve + }) + let sawUnsubscribe: (() => void) | undefined + const unsubscribed = new Promise(resolve => { + sawUnsubscribe = resolve + }) + const oldLocal = client({ hash: localHash }) + ;(oldLocal as any).subscribeReorgs = jest.fn(async () => 'sub-1') + ;(oldLocal as any).unsubscribe = jest.fn(async () => { + sawUnsubscribe!() + await pendingUnsubscribe + return true + }) + const recovered = client({ hash: agreedHash }) + const recoverLocal = jest.fn(async () => recovered) + const replacement = client() + const clearLocal = jest.fn(async () => replacement) + const tracker = new LocalChainTracker({ + local: oldLocal, + fallbacks: [client({ hash: agreedHash }), client({ hash: agreedHash })], + requiredConsistencyAgreement: 2, + autoRecover: true, + recoverLocal, + clearLocal + }) + await tracker.getVerificationContextToken() + + const recovering = tracker.checkConsistency() + await unsubscribed + const clearing = tracker.clearLocalData() + releaseUnsubscribe!() + await expect(recovering).resolves.toMatchObject({ + consistency: 'error', + lastError: 'Local ChainTracks reset was superseded' + }) + await expect(clearing).resolves.toMatchObject({ consistency: 'unchecked' }) + expect(recoverLocal).not.toHaveBeenCalled() + expect(clearLocal).toHaveBeenCalledTimes(1) + expect(tracker.getLocalClient()).toBe(replacement) + }) + + test('does not invoke a superseded clear hook when reset is overtaken during disposal', async () => { + let releaseUnsubscribe: (() => void) | undefined + const pendingUnsubscribe = new Promise(resolve => { + releaseUnsubscribe = resolve + }) + let sawUnsubscribe: (() => void) | undefined + const unsubscribed = new Promise(resolve => { + sawUnsubscribe = resolve + }) + const oldLocal = client() + ;(oldLocal as any).subscribeReorgs = jest.fn(async () => 'sub-1') + ;(oldLocal as any).unsubscribe = jest.fn(async () => { + sawUnsubscribe!() + await pendingUnsubscribe + return true + }) + const replacement = client() + const clearLocal = jest.fn(async () => replacement) + const tracker = new LocalChainTracker({ local: oldLocal, clearLocal }) + await tracker.getVerificationContextToken() + + const first = tracker.clearLocalData() + await unsubscribed + const second = tracker.clearLocalData() + releaseUnsubscribe!() + await expect(first).rejects.toThrow('superseded') + await expect(second).resolves.toMatchObject({ consistency: 'unchecked' }) + expect(clearLocal).toHaveBeenCalledTimes(1) + expect(tracker.getLocalClient()).toBe(replacement) + }) + + test('keeps the newest clear replacement when concurrent resets complete out of order', async () => { + let resolveFirst: ((value: ChaintracksClientApi) => void) | undefined + let resolveSecond: ((value: ChaintracksClientApi) => void) | undefined + const first = new Promise(resolve => { + resolveFirst = resolve + }) + const second = new Promise(resolve => { + resolveSecond = resolve + }) + let firstHookEntered: (() => void) | undefined + const firstHook = new Promise(resolve => { + firstHookEntered = resolve + }) + const older = client({ valid: false }) + const newer = client({ valid: true }) + const clearLocal = jest + .fn() + .mockImplementationOnce(async () => { + firstHookEntered!() + return await first + }) + .mockImplementationOnce(async () => await second) + const tracker = new LocalChainTracker({ local: client(), clearLocal }) + + const oldReset = tracker.clearLocalData() + await firstHook + const newReset = tracker.clearLocalData() + resolveSecond!(newer) + await newReset + resolveFirst!(older) + await expect(oldReset).rejects.toThrow('superseded') + + expect(tracker.getLocalClient()).toBe(newer) + await expect(tracker.isValidRootForHeight('root', 100)).resolves.toBe(true) + }) + test('requires a configured clearing hook and resets local-primary status when cleared', async () => { const unconfigured = new LocalChainTracker({ local: client() }) await expect(unconfigured.clearLocalData()).rejects.toThrow('Local ChainTracks clearing is not configured.') @@ -287,6 +512,48 @@ describe('LocalChainTracker', () => { }) }) + test('rejects a local validation result when the tracker mode changes in flight', async () => { + let release: (() => void) | undefined + const pending = new Promise(resolve => { + release = resolve + }) + const local = client() + ;(local.isValidRootForHeight as jest.Mock).mockImplementation(async () => { + await pending + return true + }) + const tracker = new LocalChainTracker({ local, fallbacks: [client({ valid: true })] }) + + const validation = tracker.isValidRootForHeight('root', 100) + tracker.setMode('remote-only') + release!() + + await expect(validation).rejects.toThrow('provider changed') + expect(tracker.getVerificationContext()).toContain('local-chaintracks:[1,false,"remote-only"') + }) + + test('rejects a local height result when its nested provider context changes in flight', async () => { + let release: (() => void) | undefined + const pending = new Promise(resolve => { + release = resolve + }) + let providerContext = 0 + const local = client() + ;(local.getPresentHeight as jest.Mock).mockImplementation(async () => { + await pending + return 101 + }) + ;(local as ChaintracksClientApi & { getVerificationContext: () => number }).getVerificationContext = () => + providerContext + const tracker = new LocalChainTracker({ local }) + + const height = tracker.currentHeight() + providerContext++ + release!() + + await expect(height).rejects.toThrow('provider changed') + }) + test('reports missing, unavailable, and sub-quorum consistency references', async () => { const noReferences = new LocalChainTracker({ local: client() }) await expect(noReferences.checkConsistency()).resolves.toMatchObject({ diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/ChaintracksClientApi.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/ChaintracksClientApi.ts index e41ed9773..36558dfa8 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/ChaintracksClientApi.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/ChaintracksClientApi.ts @@ -103,6 +103,13 @@ export interface ChaintracksAvailabilitySnapshotApi { * @public */ export interface ChaintracksClientApi extends ChainTracker { + /** + * Additive reorg-event capability. Method presence is not capability: + * `false` means subscribe/unsubscribe are unsupported stubs and must not be + * called; `true` promises working registration whose failures must propagate. + * Omit to infer from a callable `subscribeReorgs` implementation. + */ + readonly supportsReorgEvents?: boolean /** * Confirms the chain */ diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/ChaintracksServiceClient.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/ChaintracksServiceClient.ts index 6e0233703..236dd59cb 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/ChaintracksServiceClient.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/ChaintracksServiceClient.ts @@ -17,6 +17,8 @@ export interface ChaintracksServiceClientOptions {} * */ export class ChaintracksServiceClient implements ChaintracksClientApi { + /** HTTP polling client; callback event methods are legacy unsupported stubs. */ + readonly supportsReorgEvents = false static createChaintracksServiceClientOptions(): ChaintracksServiceClientOptions { const options: ChaintracksServiceClientOptions = { useAuthrite: false diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/GoChaintracksServiceClient.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/GoChaintracksServiceClient.ts index b0b7c8b67..19b86e53c 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/GoChaintracksServiceClient.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/GoChaintracksServiceClient.ts @@ -40,6 +40,8 @@ interface SseSubscription { * processing without a local WhatsOnChain polling ingestor. */ export class GoChaintracksServiceClient implements ChaintracksClientApi { + /** SSE client; reorg registration is a supported additive capability. */ + readonly supportsReorgEvents = true private readonly baseUrl: string private readonly fetcher: typeof fetch private readonly requestTimeoutMsecs: number diff --git a/packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.test.ts b/packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.test.ts index 0ab698888..1008c7bf7 100644 --- a/packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.test.ts +++ b/packages/wallet/wallet-toolbox/src/utility/__tests__/identityVerification.test.ts @@ -10,7 +10,13 @@ import { VerifiableCertificate } from '@bsv/sdk' import { LookupAnswer, LookupResolver } from '@bsv/sdk' -import { parseResults, parseResults$, queryOverlay } from '../identityUtils' +import { + IdentityEvidenceVerifier, + parseResults, + parseResults$, + queryOverlay, + queryOverlayEvidence +} from '../identityUtils' import { verifyOverlayOutput } from '../verifyOverlayOutput' import { createIdentityVerificationFixture, @@ -59,6 +65,30 @@ describe('identity overlay verification', () => { }) }) + it('propagates identity evidence limits instead of returning a partial result', async () => { + const receipt = answer(outputFor(fixture.certificateBEEF)) + const resolver = { query: jest.fn(async () => receipt) } as unknown as LookupResolver + await expect(queryOverlayEvidence({}, resolver, { candidateBytes: 1 })).rejects.toMatchObject({ code: 'limit' }) + await expect( + queryOverlayEvidence({}, resolver, { + candidateBytes: fixture.certificateBEEF.length, + retainedBytes: fixture.certificateBEEF.length, + outputs: 1 + }) + ).resolves.toEqual(receipt) + + const limitedResolver = { + query: jest.fn(async (_query, _auth, options) => { + options.onEvidence({ type: 'limit' }) + return receipt + }) + } as unknown as LookupResolver + await expect(queryOverlayEvidence({}, limitedResolver)).rejects.toMatchObject({ code: 'limit' }) + + const verifier = new IdentityEvidenceVerifier(fixture.confirmedTracker, 'limit-test', { candidateBytes: 1 }) + await expect(verifier.parse(outputFor(fixture.certificateBEEF))).rejects.toMatchObject({ code: 'limit' }) + }) + it('accepts a genuinely signed unconfirmed spend only when its ancestor is confirmed', async () => { await expect(fixture.unconfirmedTransaction.verify(fixture.confirmedTracker)).resolves.toBe(true) diff --git a/packages/wallet/wallet-toolbox/src/utility/identityUtils.ts b/packages/wallet/wallet-toolbox/src/utility/identityUtils.ts index dd6cf00e3..ecb847248 100644 --- a/packages/wallet/wallet-toolbox/src/utility/identityUtils.ts +++ b/packages/wallet/wallet-toolbox/src/utility/identityUtils.ts @@ -9,10 +9,15 @@ import { IdentityCertificate, IdentityCertifier, Base64String, - ChainTracker + ChainTracker, + TransactionEvidenceCoordinator, + TransactionEvidenceError, + TransactionEvidenceLimits, + VerifiedTransactionOutput, + defaultTransactionEvidenceLimits } from '@bsv/sdk' import { Certifier, TrustSettings } from '../WalletSettingsManager' -import { OverlayOutputEvidence, verifyOverlayOutput } from './verifyOverlayOutput' +import { OverlayOutputEvidence } from './verifyOverlayOutput' // Our extended certificate includes certifierInfo. export interface ExtendedVerifiableCertificate extends IdentityCertificate { @@ -114,25 +119,72 @@ export const queryOverlay = async ( return await parseResults(await queryOverlayEvidence(query, resolver), chainTracker) } -/** Fetch an owned snapshot of UNTRUSTED evidence, suitable only for revalidation. */ -export const queryOverlayEvidence = async (query: unknown, resolver: LookupResolver): Promise => { - const results = await resolver.query( - { - service: 'ls_identity', - query - }, - undefined, - { graceMs: 300 } - ) - - if (results.type !== 'output-list') return results - return { - type: 'output-list', - outputs: results.outputs.map(output => ({ +/** Configurable identity intake bounds; share byte limits with IdentityEvidenceVerifier. */ +export interface IdentityEvidenceIntakeLimits { + candidateBytes?: number + retainedBytes?: number + outputs?: number +} + +/** + * Fetch an owned snapshot of UNTRUSTED evidence, suitable only for revalidation. + * A limit rejects the lookup because this legacy result cannot represent partial completion. + */ +export const queryOverlayEvidence = async ( + query: unknown, + resolver: LookupResolver, + limits: IdentityEvidenceIntakeLimits = {} +): Promise => { + const candidateBytes = limits.candidateBytes ?? defaultTransactionEvidenceLimits.candidateBytes + const retainedBytes = limits.retainedBytes ?? defaultTransactionEvidenceLimits.retainedBytes + const maxOutputs = limits.outputs ?? 512 + if (![candidateBytes, retainedBytes, maxOutputs].every(value => Number.isSafeInteger(value) && value > 0)) + throw new TransactionEvidenceError('limit') + const outputs: LookupAnswer['outputs'] = [] + let bytes = 0 + let received = false + let closed = false + let limited = false + const accept = (output: LookupAnswer['outputs'][number]): void => { + const size = output.beef.length + (output.context?.length ?? 0) + if (outputs.length >= maxOutputs || output.beef.length > candidateBytes || bytes + size > retainedBytes) { + limited = true + return + } + bytes += size + outputs.push({ ...output, beef: output.beef.slice(), ...(output.context === undefined ? {} : { context: output.context.slice() }) - })) + }) + } + try { + const results = await resolver.query( + { + service: 'ls_identity', + query + }, + undefined, + { + graceMs: 300, + evidenceLimits: { maxOutputs, maxBytes: retainedBytes }, + onEvidence: event => { + if (closed) return + received = true + if (event.type === 'output') accept(event.output) + else limited = true + } + } + ) + + if (limited) throw new TransactionEvidenceError('limit') + if (results.type !== 'output-list') return results + // Custom/older resolvers may not implement the additive intake callback. + if (!received) for (const output of results.outputs) accept(output) + if (limited) throw new TransactionEvidenceError('limit') + return { type: 'output-list', outputs } + } finally { + closed = true } } @@ -160,12 +212,10 @@ const yieldToUi = async (): Promise => { * Parse a single overlay output into a verified, decrypted certificate. Returns `null` on any * parse / decrypt / verify failure so a malformed entry can never block the others. */ -const parseOne = async ( - output: OverlayOutputEvidence, - chainTracker: ChainTracker +const decodeIdentityOutput = async ( + verifiedOutput: VerifiedTransactionOutput ): Promise => { try { - const verifiedOutput = await verifyOverlayOutput(output, chainTracker) const decodedOutput = PushDrop.decode(verifiedOutput.lockingScript) const certificate: VerifiableCertificate = JSON.parse(Utils.toUTF8(decodedOutput.fields[0])) const verifiableCert = new VerifiableCertificate( @@ -202,6 +252,108 @@ const parseOne = async ( } } +/** + * Wallet/session-owned identity validation. Chain evidence is checked on every + * use; bounded certificate crypto results are independent of mutable trust ratings. + */ +export class IdentityEvidenceVerifier { + private disposed = false + private readonly coordinator: TransactionEvidenceCoordinator + private readonly certificates = new Map() + private expiryTimer?: ReturnType + + constructor( + readonly chainTracker: ChainTracker, + readonly chainNamespace = 'caller-chain-tracker', + limits?: Partial + ) { + this.coordinator = new TransactionEvidenceCoordinator({ + chainTracker, + chainNamespace, + limits, + policyId: 'sdk-spv-overlay-graph-v1' + }) + } + + private clone(json: string): VerifiableCertificate { + const cert: VerifiableCertificate = JSON.parse(json) + const copy = new VerifiableCertificate( + cert.type, + cert.serialNumber, + cert.subject, + cert.certifier, + cert.revocationOutpoint, + cert.fields, + cert.keyring, + cert.signature + ) + copy.decryptedFields = cert.decryptedFields + return copy + } + + async parse(output: OverlayOutputEvidence): Promise<{ outpoint: string; certificate: VerifiableCertificate } | null> { + try { + const evidence = { ...output, beef: output.beef.slice() } + const verified = await this.coordinator.verify(evidence) + if (this.disposed) return null + const cached = this.certificates.get(verified.outpoint) + if (cached !== undefined && cached.expiresAt > Date.now()) { + return { outpoint: verified.outpoint, certificate: this.clone(cached.json) } + } + const certificate = await decodeIdentityOutput(verified) + if (certificate === null) return null + // Certificate crypto can await; fence its completion against current chain/session state. + await this.coordinator.verify(evidence) + if (this.disposed) return null + // Store owned bytes and return a separate object; callers cannot mutate future results. + const json = JSON.stringify(certificate) + this.prune() + let bytes = [...this.certificates.values()].reduce((total, value) => total + value.json.length * 2, 0) + while ( + this.certificates.size > 0 && + (this.certificates.size >= 128 || bytes + json.length * 2 > 2 * 1024 * 1024) + ) { + const oldest = this.certificates.keys().next().value! + bytes -= this.certificates.get(oldest)!.json.length * 2 + this.certificates.delete(oldest) + } + if (json.length * 2 <= 2 * 1024 * 1024) { + this.certificates.set(verified.outpoint, { json, expiresAt: Date.now() + 60_000 }) + this.scheduleExpiry() + } + return { outpoint: verified.outpoint, certificate: this.clone(json) } + } catch (error) { + if (error instanceof TransactionEvidenceError && (error.code === 'limit' || error.code === 'timeout')) throw error + return null + } + } + + private prune(): void { + for (const [key, cached] of this.certificates) if (cached.expiresAt <= Date.now()) this.certificates.delete(key) + } + + private scheduleExpiry(): void { + clearTimeout(this.expiryTimer) + const expiresAt = Math.min(...[...this.certificates.values()].map(value => value.expiresAt)) + if (!Number.isFinite(expiresAt)) return + this.expiryTimer = setTimeout( + () => { + this.prune() + this.scheduleExpiry() + }, + Math.max(1, expiresAt - Date.now()) + ) + this.expiryTimer.unref?.() + } + + dispose(): void { + this.disposed = true + this.coordinator.dispose() + this.certificates.clear() + clearTimeout(this.expiryTimer) + } +} + /** * Parse the returned UTXOs, decrypting and verifying each certificate. * An omitted ChainTracker fails closed. Each call revalidates transaction evidence; @@ -212,17 +364,12 @@ const parseOne = async ( */ export const parseResults = async ( lookupResult: LookupAnswer, - chainTracker?: ChainTracker + chainTracker?: ChainTracker, + verifier?: IdentityEvidenceVerifier ): Promise => { - if (lookupResult.type !== 'output-list' || chainTracker == null) return [] - const parsedResults: VerifiableCertificate[] = [] - const shouldYield = isUiRuntime() - for (const output of lookupResult.outputs) { - if (shouldYield) await yieldToUi() - const cert = await parseOne(output, chainTracker) - if (cert != null) parsedResults.push(cert) - } - return parsedResults + const certificates: VerifiableCertificate[] = [] + for await (const certificate of parseResults$(lookupResult, chainTracker, verifier)) certificates.push(certificate) + return certificates } /** @@ -231,13 +378,25 @@ export const parseResults = async ( */ export async function* parseResults$( lookupResult: LookupAnswer, - chainTracker?: ChainTracker + chainTracker?: ChainTracker, + verifier?: IdentityEvidenceVerifier ): AsyncIterable { if (lookupResult.type !== 'output-list' || chainTracker == null) return + if (verifier !== undefined && verifier.chainTracker !== chainTracker) return + const owned = verifier === undefined + verifier ??= new IdentityEvidenceVerifier(chainTracker) const shouldYield = isUiRuntime() - for (const output of lookupResult.outputs) { - if (shouldYield) await yieldToUi() - const cert = await parseOne(output, chainTracker) - if (cert != null) yield cert + const seen = new Set() + try { + for (const output of lookupResult.outputs) { + if (shouldYield) await yieldToUi() + const result = await verifier.parse(output) + if (result !== null && !seen.has(result.outpoint)) { + seen.add(result.outpoint) + yield result.certificate + } + } + } finally { + if (owned) verifier.dispose() } } diff --git a/packages/wallet/wallet-toolbox/src/utility/verifyOverlayOutput.ts b/packages/wallet/wallet-toolbox/src/utility/verifyOverlayOutput.ts index ea2cac8be..7681f4a83 100644 --- a/packages/wallet/wallet-toolbox/src/utility/verifyOverlayOutput.ts +++ b/packages/wallet/wallet-toolbox/src/utility/verifyOverlayOutput.ts @@ -1,79 +1,28 @@ -import { ChainTracker, LockingScript, Transaction } from '@bsv/sdk' +import { ChainTracker, TransactionEvidence, TransactionEvidenceCoordinator, VerifiedTransactionOutput } from '@bsv/sdk' /** Untrusted lookup evidence. A host's txid and context are never chain authority. */ -export interface OverlayOutputEvidence { - beef: number[] - outputIndex: number - txid?: string +export interface OverlayOutputEvidence extends TransactionEvidence { context?: number[] } -/** A per-call verdict, for service validation now; not a reusable chain/cache verdict. */ -export interface VerifiedOverlayOutput { - readonly txid: string - readonly outputIndex: number - readonly lockingScript: LockingScript -} +export type VerifiedOverlayOutput = VerifiedTransactionOutput /** - * Independently verify a byte-bound output using the caller's canonical chain source. - * No network defaults, scripts-only mode, verdict cache, or identity semantics live here. - * Errors and false verdicts reject this evidence candidate, not its txid permanently. + * Independently verify one byte-bound output using the caller's canonical chain + * source. Repeated consumers can use the SDK TransactionEvidenceCoordinator. */ export async function verifyOverlayOutput( evidence: OverlayOutputEvidence, chainTracker: ChainTracker ): Promise { - if (chainTracker == null || typeof chainTracker.isValidRootForHeight !== 'function') { - throw new Error('Overlay verification requires a ChainTracker') - } - const { outputIndex, txid: hint } = evidence - if (!Number.isSafeInteger(outputIndex) || outputIndex < 0) { - throw new Error('Invalid overlay output index') - } - // Own the bytes before the first await: callers cannot change the candidate in flight. - const bytes = evidence.beef.slice() - if (bytes.some(byte => !Number.isInteger(byte) || byte < 0 || byte > 255)) { - throw new Error('Invalid overlay transaction bytes') - } - const tx = Transaction.fromBEEF(bytes) - const txid = tx.id('hex') - if (hint !== undefined && (typeof hint !== 'string' || hint.toLowerCase() !== txid)) { - throw new Error('Overlay txid does not match transaction bytes') - } - if (outputIndex >= tx.outputs.length) throw new Error('Overlay output does not exist') - - // SDK graph verification checks scripts and values. Also require every unconfirmed - // branch to have inputs, so a fabricated zero-value, input-free leaf cannot anchor it. - const pending = [tx] - const visited = new Set() - const spentOutpoints = new Set() - while (pending.length > 0) { - const current = pending.pop()! - const currentTxid = current.id('hex') - if (visited.has(currentTxid)) continue - visited.add(currentTxid) - if (current.merklePath != null) continue - if (current.inputs.length === 0) throw new Error('Unconfirmed transaction has no ancestry') - for (const input of current.inputs) { - if (input.sourceTransaction == null) throw new Error('Overlay transaction ancestry is missing') - const sourceTxid = input.sourceTransaction.id('hex') - if (input.sourceTXID !== undefined && input.sourceTXID !== sourceTxid) { - throw new Error('Overlay input does not match its source transaction') - } - const outpoint = `${sourceTxid}.${input.sourceOutputIndex}` - // Script verification alone can count the same value twice. Reject both - // duplicate inputs and conflicting spends among unconfirmed ancestors. - if (spentOutpoints.has(outpoint)) throw new Error('Overlay transaction graph spends an outpoint twice') - spentOutpoints.add(outpoint) - pending.push(input.sourceTransaction) - } - } - - const canonicalTracker: ChainTracker = { - currentHeight: async () => await chainTracker.currentHeight(), - isValidRootForHeight: async (root, height) => (await chainTracker.isValidRootForHeight(root, height)) === true + const coordinator = new TransactionEvidenceCoordinator({ + chainTracker, + chainNamespace: 'caller-chain-tracker', + policyId: 'sdk-spv-overlay-graph-v1' + }) + try { + return await coordinator.verify(evidence) + } finally { + coordinator.dispose() } - if ((await tx.verify(canonicalTracker)) !== true) throw new Error('Overlay transaction verification failed') - return { txid, outputIndex, lockingScript: tx.outputs[outputIndex].lockingScript } } From e2d698069b453a35a00087ffce2dc04055c70f61 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 15:00:43 -0500 Subject: [PATCH 008/168] feat(overlay): harden BASM protocol validation and Go interop Validate untrusted BASM peer JSON, bind remote anchors to local canonical headers, and independently check admitted-list block indices before historical admission. Keep automatic BASM sync disabled and leave durable recovery jobs for B02. Include independent BRC-136 vector checks and localhost TS client tests against the Go five-method read/serving surface. --- .../overlays/overlay-discovery-services.md | 2 +- docs/packages/overlays/overlay-express.md | 2 +- docs/packages/overlays/overlay-topics.md | 2 +- docs/packages/overlays/overlay.md | 2 +- docs/reference/package-api-migrations.md | 24 +- docs/reference/stack-facts.md | 8 +- governance/package-release-notes.json | 16 +- governance/repository-health/baselines.json | 8 +- .../overlay-discovery-services/CHANGELOG.md | 3 + .../overlay-discovery-services/package.json | 2 +- .../overlays/overlay-express/CHANGELOG.md | 6 + packages/overlays/overlay-express/README.md | 14 + .../overlays/overlay-express/package.json | 2 +- .../overlay-express/src/OverlayExpress.ts | 115 ++++-- .../src/__tests__/OverlayExpress.test.ts | 159 +++++++- packages/overlays/overlay/CHANGELOG.md | 7 + packages/overlays/overlay/README.md | 45 +++ .../overlays/overlay/docs/BRC-136-BASM.md | 8 + packages/overlays/overlay/package.json | 2 +- packages/overlays/overlay/src/BASM.ts | 15 +- packages/overlays/overlay/src/BASMRemote.ts | 247 +++++++++++- .../overlays/overlay/src/BASMValidation.ts | 131 ++++++ packages/overlays/overlay/src/Engine.ts | 136 ++++++- .../src/__tests/BASM.conformance.test.ts | 165 ++++++++ .../overlay/src/__tests/BASMGoInterop.test.ts | 214 ++++++++++ .../src/__tests/BASMReconciliation.test.ts | 357 ++++++++++++++++ .../overlay/src/__tests/BASMRemote.test.ts | 380 ++++++++++++++++++ .../overlay/src/__tests/fixtures/README.md | 26 ++ .../__tests/fixtures/basm-go-read-server.go | 211 ++++++++++ .../__tests/fixtures/brc136-independent.json | 185 +++++++++ packages/overlays/topics/CHANGELOG.md | 3 + packages/overlays/topics/package.json | 2 +- 32 files changed, 2395 insertions(+), 104 deletions(-) create mode 100644 packages/overlays/overlay/src/BASMValidation.ts create mode 100644 packages/overlays/overlay/src/__tests/BASM.conformance.test.ts create mode 100644 packages/overlays/overlay/src/__tests/BASMGoInterop.test.ts create mode 100644 packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts create mode 100644 packages/overlays/overlay/src/__tests/BASMRemote.test.ts create mode 100644 packages/overlays/overlay/src/__tests/fixtures/README.md create mode 100644 packages/overlays/overlay/src/__tests/fixtures/basm-go-read-server.go create mode 100644 packages/overlays/overlay/src/__tests/fixtures/brc136-independent.json diff --git a/docs/packages/overlays/overlay-discovery-services.md b/docs/packages/overlays/overlay-discovery-services.md index 8400631f9..125f6920c 100644 --- a/docs/packages/overlays/overlay-discovery-services.md +++ b/docs/packages/overlays/overlay-discovery-services.md @@ -4,7 +4,7 @@ title: '@bsv/overlay-discovery-services' kind: package domain: overlays npm: '@bsv/overlay-discovery-services' -version: '2.2.1' +version: '2.2.2' last_updated: '2026-08-27' last_verified: '2026-08-27' review_cadence_days: 30 diff --git a/docs/packages/overlays/overlay-express.md b/docs/packages/overlays/overlay-express.md index 2c4880412..306ef3cd9 100644 --- a/docs/packages/overlays/overlay-express.md +++ b/docs/packages/overlays/overlay-express.md @@ -4,7 +4,7 @@ title: '@bsv/overlay-express' kind: package domain: overlays npm: '@bsv/overlay-express' -version: '2.6.1' +version: '2.7.0' last_updated: '2026-08-27' last_verified: '2026-08-27' review_cadence_days: 30 diff --git a/docs/packages/overlays/overlay-topics.md b/docs/packages/overlays/overlay-topics.md index 691a86814..226a99019 100644 --- a/docs/packages/overlays/overlay-topics.md +++ b/docs/packages/overlays/overlay-topics.md @@ -4,7 +4,7 @@ title: '@bsv/overlay-topics' kind: package domain: overlays npm: '@bsv/overlay-topics' -version: '1.7.1' +version: '1.7.2' last_updated: '2026-08-27' last_verified: '2026-08-27' review_cadence_days: 30 diff --git a/docs/packages/overlays/overlay.md b/docs/packages/overlays/overlay.md index 2a4492822..14c6cfb05 100644 --- a/docs/packages/overlays/overlay.md +++ b/docs/packages/overlays/overlay.md @@ -4,7 +4,7 @@ title: '@bsv/overlay' kind: package domain: overlays npm: '@bsv/overlay' -version: '2.3.1' +version: '2.4.0' last_updated: '2026-08-27' last_verified: '2026-08-27' review_cadence_days: 30 diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 6f775f6ee..a1b4b94e4 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -42,10 +42,10 @@ and clean-consumer tests remain the executable type authority. | `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | | `@bsv/message-box-client` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | -| `@bsv/overlay` | `2.2.1` | `2.3.1` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.1` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-express` | `2.5.0` | `2.6.1` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-topics` | `1.6.10` | `1.7.1` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. | +| `@bsv/overlay` | `2.2.1` | `2.4.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. Existing lookup limits and abort-hook guidance remain applicable. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.2` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update. | +| `@bsv/overlay-express` | `2.5.0` | `2.7.0` | minor | [API and usage](../packages/overlays/overlay-express.md) | Valid legacy JSON shapes, numeric-string heights, public CORS, topic-free raw transactions, configured limits and synchronization defaults remain supported. No storage migration is required. Malformed/coercive inputs are rejected and missing BASM engine/storage capabilities return HTTP 400 with code BASM_UNSUPPORTED. Existing header providers remain valid without a blockTransactionCount field. TTN setup and license-notice requirements remain unchanged. | +| `@bsv/overlay-topics` | `1.6.10` | `1.7.2` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. No migration is required for this dependency candidate update. | | `@bsv/paymail` | `2.4.2` | `2.4.7` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. Consumers of the former bundled Money Button or Tokenized specification documents must follow the authoritative links in docs/specs/README.md. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/payment-express-middleware` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/sdk` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/sdk/bsv-sdk.md) | No API or valid BUMP wire migration is required for the offset correction. Historical number-array fast paths and React Native behavior remain compatible. Documentation users should load docs/swagger/swagger.yaml into their preferred viewer instead of using the removed static Swagger UI scaffold. Distributors must keep THIRD_PARTY_NOTICES.md and LICENSES/ with source and browser bundles. | @@ -277,8 +277,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay.md](../packages/overlays/overlay.md) - Source: [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) -- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. -- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. +- Release note: Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Retains the engine lookup cardinality ceiling, optional TopicManager abort hook and scoped pre-uniformization Open BSV License version 4 continuity notice. +- Migration: Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. Existing lookup limits and abort-hook guidance remain applicable. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | @@ -292,8 +292,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-discovery-services.md](../packages/overlays/overlay-discovery-services.md) - Source: [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) -- Release note: Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. -- Migration: Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. +- Release note: Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Advances the packed overlay dependency candidate for BASM validation hardening without changing this package runtime behavior. +- Migration: Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------------- | ---------------------------------------------------- | @@ -304,8 +304,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-express.md](../packages/overlays/overlay-express.md) - Source: [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) -- Release note: Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. -- Migration: Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. +- Release note: Validates BASM heights, hashes and txid sets; exposes unsupported capabilities explicitly and accepts an optional trusted full-block count from injected canonical header resolvers. Retains TerraTestNet runtime support, TTN discovery/advertising, explicit ChainTracks requirements, Arcade-first propagation, and the scoped pre-uniformization Open BSV License version 4 continuity notice. +- Migration: Valid legacy JSON shapes, numeric-string heights, public CORS, topic-free raw transactions, configured limits and synchronization defaults remain supported. No storage migration is required. Malformed/coercive inputs are rejected and missing BASM engine/storage capabilities return HTTP 400 with code BASM_UNSUPPORTED. Existing header providers remain valid without a blockTransactionCount field. TTN setup and license-notice requirements remain unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------------- | ---------------------------------------------------- | @@ -316,8 +316,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-topics.md](../packages/overlays/overlay-topics.md) - Source: [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) -- Release note: Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. -- Migration: Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. +- Release note: Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. Advances the packed overlay dependency candidate for BASM validation hardening without changing this package runtime behavior. +- Migration: Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. No migration is required for this dependency candidate update. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | -------------------------------------- | --------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 5e6e51522..5a6cea775 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -58,10 +58,10 @@ authorized release action. | network | `@bsv/chirp` | `0.1.1` | browser-library | browser-bundler, browser-esm, cli, node-esm | browser, node | `>=22` | [packages/network/chirp](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/chirp) | | network | `@bsv/teranode-listener` | `1.1.5` | node-library | node-esm | node | `>=22` | [packages/network/ts-p2p](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/ts-p2p) | | overlays | `@bsv/gasp` | `1.3.6` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/overlays/gasp-core](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/gasp-core) | -| overlays | `@bsv/overlay` | `2.3.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | -| overlays | `@bsv/overlay-discovery-services` | `2.2.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | -| overlays | `@bsv/overlay-express` | `2.6.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | -| overlays | `@bsv/overlay-topics` | `1.7.1` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | +| overlays | `@bsv/overlay` | `2.4.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | +| overlays | `@bsv/overlay-discovery-services` | `2.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | +| overlays | `@bsv/overlay-express` | `2.7.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | +| overlays | `@bsv/overlay-topics` | `1.7.2` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | | sdk | `@bsv/sdk` | `2.5.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | | sdk | `@bsv/verifast` | `0.3.5` | wasm-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global, wasm-worker | browser, node, umd, wasm, worker | `>=22` | [packages/verifast](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/verifast) | | wallet | `@bsv/btms` | `1.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index 65bd5ae07..b4dba89e8 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -126,29 +126,29 @@ "name": "@bsv/overlay", "publishedVersion": "2.2.1", "releaseType": "minor", - "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice.", - "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package." + "summary": "Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Retains the engine lookup cardinality ceiling, optional TopicManager abort hook and scoped pre-uniformization Open BSV License version 4 continuity notice.", + "migration": "Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. Existing lookup limits and abort-hook guidance remain applicable. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package." }, { "name": "@bsv/overlay-discovery-services", "publishedVersion": "2.1.1", "releaseType": "minor", - "summary": "Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice.", - "migration": "Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package." + "summary": "Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Advances the packed overlay dependency candidate for BASM validation hardening without changing this package runtime behavior.", + "migration": "Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update." }, { "name": "@bsv/overlay-express", "publishedVersion": "2.5.0", "releaseType": "minor", - "summary": "Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice.", - "migration": "Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package." + "summary": "Validates BASM heights, hashes and txid sets; exposes unsupported capabilities explicitly and accepts an optional trusted full-block count from injected canonical header resolvers. Retains TerraTestNet runtime support, TTN discovery/advertising, explicit ChainTracks requirements, Arcade-first propagation, and the scoped pre-uniformization Open BSV License version 4 continuity notice.", + "migration": "Valid legacy JSON shapes, numeric-string heights, public CORS, topic-free raw transactions, configured limits and synchronization defaults remain supported. No storage migration is required. Malformed/coercive inputs are rejected and missing BASM engine/storage capabilities return HTTP 400 with code BASM_UNSUPPORTED. Existing header providers remain valid without a blockTransactionCount field. TTN setup and license-notice requirements remain unchanged." }, { "name": "@bsv/overlay-topics", "publishedVersion": "1.6.10", "releaseType": "minor", - "summary": "Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name.", - "migration": "Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted." + "summary": "Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. Advances the packed overlay dependency candidate for BASM validation hardening without changing this package runtime behavior.", + "migration": "Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. No migration is required for this dependency candidate update." }, { "name": "@bsv/paymail", diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index aebb1eaf6..1890113c1 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -318,10 +318,10 @@ "@bsv/payment-express-middleware": "2.1.6", "@bsv/teranode-listener": "1.1.5", "@bsv/gasp": "1.3.6", - "@bsv/overlay": "2.3.1", - "@bsv/overlay-discovery-services": "2.2.1", - "@bsv/overlay-express": "2.6.1", - "@bsv/overlay-topics": "1.7.1", + "@bsv/overlay": "2.4.0", + "@bsv/overlay-discovery-services": "2.2.2", + "@bsv/overlay-express": "2.7.0", + "@bsv/overlay-topics": "1.7.2", "@bsv/sdk": "2.5.0", "@bsv/verifast": "0.3.5", "@bsv/btms": "1.2.2", diff --git a/packages/overlays/overlay-discovery-services/CHANGELOG.md b/packages/overlays/overlay-discovery-services/CHANGELOG.md index eb62a7421..67881e071 100644 --- a/packages/overlays/overlay-discovery-services/CHANGELOG.md +++ b/packages/overlays/overlay-discovery-services/CHANGELOG.md @@ -9,6 +9,9 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +- Advances the packed overlay dependency candidate for BASM validation hardening. + Package runtime behavior is unchanged; no consumer migration is required. + ### Added - Support `ttn` WalletAdvertiser instances with chain-isolated TerraTestNet diff --git a/packages/overlays/overlay-discovery-services/package.json b/packages/overlays/overlay-discovery-services/package.json index f03df3f92..610bb5080 100644 --- a/packages/overlays/overlay-discovery-services/package.json +++ b/packages/overlays/overlay-discovery-services/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-discovery-services", - "version": "2.2.1", + "version": "2.2.2", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/overlay-express/CHANGELOG.md b/packages/overlays/overlay-express/CHANGELOG.md index fc6b08174..28c7a3cd5 100644 --- a/packages/overlays/overlay-express/CHANGELOG.md +++ b/packages/overlays/overlay-express/CHANGELOG.md @@ -11,6 +11,12 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +- Validated BASM heights, hashes and txid sets while retaining numeric-string + heights, public CORS, topic-free raw transactions, configured limits and sync + defaults. Added explicit unsupported-capability codes and optional trusted + full-block transaction counts on injected header resolvers. No deployment or + storage migration is required. + ### Added - Added the `ttn` runtime network, TTN SHIP/SLAP roots, and explicit diff --git a/packages/overlays/overlay-express/README.md b/packages/overlays/overlay-express/README.md index ffa615e9d..b75f11aed 100644 --- a/packages/overlays/overlay-express/README.md +++ b/packages/overlays/overlay-express/README.md @@ -36,6 +36,20 @@ import OverlayExpress, { OverlayMonitor } from '@bsv/overlay-express' const { default: OverlayExpress, OverlayMonitor } = require('@bsv/overlay-express') ``` +The five public BASM JSON POST routes validate nonnegative safe-integer heights +(including existing numeric strings), 32-byte hexadecimal hashes/txids, and +request count limits. Empty raw-transaction requests remain valid; compound +proof requests require txids. The raw-transaction route does not require +`x-bsv-topic`. Missing engine/storage BASM capabilities retain HTTP 400 with +`{ status: 'error', message, code: 'BASM_UNSUPPORTED' }`. CORS, access rules, +configured limits, and automatic synchronization defaults are unchanged. + +An injected topic-anchor header resolver may additionally return +`blockTransactionCount` obtained independently for the same canonical block +hash. Existing Chaintracks/provider adapters remain header-only; they do not +claim this stronger position evidence. See the core engine's +[BASM validation and recovery limits](../overlay/README.md#basm-peer-validation-and-current-recovery-limits). + ## Example Usage Here's a quick example: diff --git a/packages/overlays/overlay-express/package.json b/packages/overlays/overlay-express/package.json index 9b45f02cc..27c767b98 100644 --- a/packages/overlays/overlay-express/package.json +++ b/packages/overlays/overlay-express/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-express", - "version": "2.6.1", + "version": "2.7.0", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/overlay-express/src/OverlayExpress.ts b/packages/overlays/overlay-express/src/OverlayExpress.ts index 888d78d31..24ab06ae9 100644 --- a/packages/overlays/overlay-express/src/OverlayExpress.ts +++ b/packages/overlays/overlay-express/src/OverlayExpress.ts @@ -206,6 +206,8 @@ export type TopicAnchorHeaderResolver = (blockHeight: number) => Promise< blockHeight: number blockHash: string merkleRoot?: string + /** Independent full block count bound to blockHash; never an overlay subset count. */ + blockTransactionCount?: number } | undefined > @@ -251,6 +253,15 @@ class PublicRequestError extends Error { } } +class UnsupportedBasmCapabilityError extends PublicRequestError { + readonly code = 'BASM_UNSUPPORTED' + + constructor() { + super('BASM capability is not supported by this Overlay engine') + this.name = 'UnsupportedBasmCapabilityError' + } +} + function publicErrorMessage( error: unknown, fallback: string = 'Request could not be processed' @@ -2249,6 +2260,13 @@ export default class OverlayExpress { return res.status(200).json(await handler(req)) } catch (error) { console.error(chalk.red(`Error in ${path}:`), error) + if (error instanceof Error && 'code' in error && error.code === 'BASM_UNSUPPORTED') { + return res.status(400).json({ + status: 'error', + message: 'BASM capability is not supported by this Overlay engine', + code: error.code + }) + } return res.status(400).json({ status: 'error', message: publicErrorMessage(error) @@ -2261,26 +2279,73 @@ export default class OverlayExpress { ) } - const requireTxids = (value: unknown): string[] => { - if (!Array.isArray(value) || !value.every(txid => typeof txid === 'string')) { - throw new PublicRequestError('txids must be an array of strings') + type BasmCapability = + | 'provideTopicAnchorTip' + | 'provideTopicAnchorRange' + | 'provideAdmittedList' + | 'provideCompoundMerklePath' + | 'provideRawTransactions' + const requireBasmCapability = (capability: BasmCapability): void => { + if (typeof (engine as Partial)[capability] !== 'function') { + throw new UnsupportedBasmCapabilityError() + } + } + + const requireBasmHeight = (value: unknown, field: string): number => { + if ( + (typeof value !== 'number' && typeof value !== 'string') || + (typeof value === 'string' && value.trim().length === 0) + ) { + throw new PublicRequestError(`${field} must be a nonnegative safe integer`) + } + const height = Number(value) + if (!Number.isSafeInteger(height) || height < 0) { + throw new PublicRequestError(`${field} must be a nonnegative safe integer`) + } + return height + } + + const requireBlockHash = (value: unknown): string | undefined => { + if (value === undefined) return undefined + if (typeof value !== 'string' || !/^[0-9a-fA-F]{64}$/.test(value)) { + throw new PublicRequestError('blockHash must be a 32-byte hexadecimal string') + } + return value.toLowerCase() + } + + const requireTxids = (value: unknown, requireAtLeastOne: boolean = true): string[] => { + if (!Array.isArray(value) || (requireAtLeastOne && value.length === 0)) { + throw new PublicRequestError('txids must be a non-empty array') } if (maxBasmTxids !== -1 && value.length > maxBasmTxids) { throw new PublicRequestError(`txids must contain at most ${maxBasmTxids} entries`) } - return value + const seen = new Set() + const txids: string[] = [] + for (const txid of value) { + if (typeof txid !== 'string' || !/^[0-9a-fA-F]{64}$/.test(txid)) { + throw new PublicRequestError('txids must contain 32-byte hexadecimal transaction IDs') + } + const normalized = txid.toLowerCase() + if (seen.has(normalized)) { + throw new PublicRequestError('txids must not contain duplicates') + } + seen.add(normalized) + txids.push(normalized) + } + return txids } - registerJsonRoute( - '/requestTopicAnchorTip', - async req => await basmEngine.provideTopicAnchorTip(readBasmTopic(req)) - ) + registerJsonRoute('/requestTopicAnchorTip', async req => { + requireBasmCapability('provideTopicAnchorTip') + return await basmEngine.provideTopicAnchorTip(readBasmTopic(req)) + }) registerJsonRoute('/requestTopicAnchorRange', async req => { const { fromHeight, toHeight } = req.body - const from = Number(fromHeight) - const to = Number(toHeight) - if (!Number.isSafeInteger(from) || !Number.isSafeInteger(to) || from < 0 || to < from) { + const from = requireBasmHeight(fromHeight, 'fromHeight') + const to = requireBasmHeight(toHeight, 'toHeight') + if (to < from) { throw new PublicRequestError('fromHeight and toHeight must define a valid ascending range') } if (maxBasmAnchorRange !== -1 && to - from + 1 > maxBasmAnchorRange) { @@ -2288,32 +2353,32 @@ export default class OverlayExpress { `topic anchor range must contain at most ${maxBasmAnchorRange} blocks` ) } + requireBasmCapability('provideTopicAnchorRange') return await basmEngine.provideTopicAnchorRange(readBasmTopic(req), from, to) }) registerJsonRoute('/requestAdmittedList', async req => { const { blockHeight, blockHash } = req.body - return await basmEngine.provideAdmittedList( - readBasmTopic(req), - Number(blockHeight), - typeof blockHash === 'string' ? blockHash : undefined - ) + const height = requireBasmHeight(blockHeight, 'blockHeight') + const hash = requireBlockHash(blockHash) + requireBasmCapability('provideAdmittedList') + return await basmEngine.provideAdmittedList(readBasmTopic(req), height, hash) }) registerJsonRoute('/requestCompoundMerklePath', async req => { const topic = readBasmTopic(req) const { blockHeight, txids } = req.body - return await basmEngine.provideCompoundMerklePath( - topic, - Number(blockHeight), - requireTxids(txids) - ) + const height = requireBasmHeight(blockHeight, 'blockHeight') + const requestedTxids = requireTxids(txids) + requireBasmCapability('provideCompoundMerklePath') + return await basmEngine.provideCompoundMerklePath(topic, height, requestedTxids) }) - registerJsonRoute( - '/requestRawTransactions', - async req => await basmEngine.provideRawTransactions(requireTxids(req.body.txids)) - ) + registerJsonRoute('/requestRawTransactions', async req => { + const txids = requireTxids(req.body.txids, false) + requireBasmCapability('provideRawTransactions') + return await basmEngine.provideRawTransactions(txids) + }) /** * ============== ADMIN ROUTES ============== diff --git a/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts b/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts index 3cb087802..363a44be8 100644 --- a/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts +++ b/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts @@ -94,6 +94,7 @@ describe('OverlayExpress', () => { expect(overlayExpress.port).toBe(3000) expect(overlayExpress.network).toBe('main') expect(overlayExpress.enableGASPSync).toBe(true) + expect(overlayExpress.enableBASMSync).toBe(false) expect(overlayExpress.verboseRequestLogging).toBe(false) expect(overlayExpress.managers).toEqual({}) expect(overlayExpress.services).toEqual({}) @@ -1692,20 +1693,49 @@ describe('OverlayExpress', () => { }) await invokeCapturedRoute(postSpy, '/requestAdmittedList', { ...topicRequest, - body: { blockHeight: 2, blockHash: 'hash' } + body: { blockHeight: '2', blockHash: 'aa'.repeat(32) } }) await invokeCapturedRoute(postSpy, '/requestCompoundMerklePath', { ...topicRequest, - body: { blockHeight: 2, txids: ['01', '02'] } + body: { blockHeight: '2', txids: ['01'.repeat(32), '02'.repeat(32)] } }) await invokeCapturedRoute(postSpy, '/requestRawTransactions', { - body: { txids: ['01'] } + body: { txids: ['01'.repeat(32)] } }) expect(mockEngine.provideTopicAnchorTip).toHaveBeenCalledWith('tm_test') expect(mockEngine.provideTopicAnchorRange).toHaveBeenCalledWith('tm_test', 1, 3) - expect(mockEngine.provideAdmittedList).toHaveBeenCalledWith('tm_test', 2, 'hash') - expect(mockEngine.provideCompoundMerklePath).toHaveBeenCalledWith('tm_test', 2, ['01', '02']) - expect(mockEngine.provideRawTransactions).toHaveBeenCalledWith(['01']) + expect(mockEngine.provideAdmittedList).toHaveBeenCalledWith('tm_test', 2, 'aa'.repeat(32)) + expect(mockEngine.provideCompoundMerklePath).toHaveBeenCalledWith('tm_test', 2, [ + '01'.repeat(32), + '02'.repeat(32) + ]) + expect(mockEngine.provideRawTransactions).toHaveBeenCalledWith(['01'.repeat(32)]) + + const tipRoute = postSpy.mock.calls.find( + (call: any[]) => call[0] === '/requestTopicAnchorTip' + ) + const rawRoute = postSpy.mock.calls.find( + (call: any[]) => call[0] === '/requestRawTransactions' + ) + const adminRoute = postSpy.mock.calls.find( + (call: any[]) => call[0] === '/admin/startBASMSync' + ) + expect(tipRoute).toHaveLength(2) + expect(rawRoute).toHaveLength(2) + expect(adminRoute.length).toBeGreaterThan(2) + + await invokeCapturedRoute(postSpy, '/requestAdmittedList', { + ...topicRequest, + body: { blockHeight: 3, blockHash: 'BB'.repeat(32) } + }) + await invokeCapturedRoute(postSpy, '/requestCompoundMerklePath', { + ...topicRequest, + body: { blockHeight: 3, txids: ['AB'.repeat(32)] } + }) + expect(mockEngine.provideAdmittedList).toHaveBeenCalledWith('tm_test', 3, 'bb'.repeat(32)) + expect(mockEngine.provideCompoundMerklePath).toHaveBeenCalledWith('tm_test', 3, [ + 'ab'.repeat(32) + ]) for (const [path, request] of [ ['/requestTopicAnchorTip', { headers: {} }], @@ -1722,6 +1752,123 @@ describe('OverlayExpress', () => { consoleError.mockRestore() }) + it('rejects malformed BASM JSON requests before invoking the engine', async () => { + const consoleError = jest.spyOn(console, 'error').mockImplementation(() => {}) + const { postSpy } = await startAndCaptureRoutes() + const topicRequest = { headers: { 'x-bsv-topic': 'tm_test' } } + + for (const [path, request, message] of [ + [ + '/requestTopicAnchorRange', + { ...topicRequest, body: { fromHeight: null, toHeight: 1 } }, + 'fromHeight must be a nonnegative safe integer' + ], + [ + '/requestTopicAnchorRange', + { ...topicRequest, body: { fromHeight: false, toHeight: 1 } }, + 'fromHeight must be a nonnegative safe integer' + ], + [ + '/requestAdmittedList', + { ...topicRequest, body: { blockHeight: {}, blockHash: 'aa'.repeat(32) } }, + 'blockHeight must be a nonnegative safe integer' + ], + [ + '/requestAdmittedList', + { ...topicRequest, body: { blockHeight: '2', blockHash: 'not-a-hash' } }, + 'blockHash must be a 32-byte hexadecimal string' + ], + [ + '/requestCompoundMerklePath', + { ...topicRequest, body: { blockHeight: '', txids: ['01'.repeat(32)] } }, + 'blockHeight must be a nonnegative safe integer' + ], + [ + '/requestCompoundMerklePath', + { ...topicRequest, body: { blockHeight: 2, txids: [] } }, + 'txids must be a non-empty array' + ], + [ + '/requestRawTransactions', + { body: { txids: ['not-a-txid'] } }, + 'txids must contain 32-byte hexadecimal transaction IDs' + ], + [ + '/requestRawTransactions', + { body: { txids: ['01'.repeat(32), '01'.repeat(32).toUpperCase()] } }, + 'txids must not contain duplicates' + ] + ] as const) { + const response = await invokeCapturedRoute(postSpy, path, request) + expect(response.status).toHaveBeenCalledWith(400) + expect(response.json).toHaveBeenCalledWith({ status: 'error', message }) + } + + expect(mockEngine.provideTopicAnchorRange).not.toHaveBeenCalled() + expect(mockEngine.provideAdmittedList).not.toHaveBeenCalled() + expect(mockEngine.provideCompoundMerklePath).not.toHaveBeenCalled() + expect(mockEngine.provideRawTransactions).not.toHaveBeenCalled() + consoleError.mockRestore() + }) + + it('preserves empty BASM responses and reports unsupported capabilities', async () => { + const consoleError = jest.spyOn(console, 'error').mockImplementation(() => {}) + const emptyTip = { + topic: 'tm_test', + blockHeight: -1, + tac: '00'.repeat(32) + } + mockEngine.provideTopicAnchorTip.mockResolvedValue(emptyTip) + const { postSpy } = await startAndCaptureRoutes() + + const tip = await invokeCapturedRoute(postSpy, '/requestTopicAnchorTip', { + headers: { 'x-bsv-topic': 'tm_test' } + }) + expect(tip.status).toHaveBeenCalledWith(200) + expect(tip.json).toHaveBeenCalledWith(emptyTip) + + const raw = await invokeCapturedRoute(postSpy, '/requestRawTransactions', { + body: { txids: [] } + }) + expect(raw.status).toHaveBeenCalledWith(200) + expect(mockEngine.provideRawTransactions).toHaveBeenCalledWith([]) + + delete mockEngine.provideRawTransactions + const unsupported = await invokeCapturedRoute(postSpy, '/requestRawTransactions', { + body: { txids: ['01'.repeat(32)] } + }) + expect(unsupported.status).toHaveBeenCalledWith(400) + expect(unsupported.json).toHaveBeenCalledWith({ + status: 'error', + message: 'BASM capability is not supported by this Overlay engine', + code: 'BASM_UNSUPPORTED' + }) + consoleError.mockRestore() + }) + + it('honors the unlimited BASM transaction limit override', async () => { + const previousLimit = process.env.OVERLAY_MAX_BASM_TXIDS + process.env.OVERLAY_MAX_BASM_TXIDS = '-1' + try { + const { postSpy } = await startAndCaptureRoutes() + const txids = Array.from({ length: 1001 }, (_, index) => + index.toString(16).padStart(64, '0') + ) + const response = await invokeCapturedRoute(postSpy, '/requestRawTransactions', { + body: { txids } + }) + + expect(response.status).toHaveBeenCalledWith(200) + expect(mockEngine.provideRawTransactions).toHaveBeenCalledWith(txids) + } finally { + if (previousLimit === undefined) { + delete process.env.OVERLAY_MAX_BASM_TXIDS + } else { + process.env.OVERLAY_MAX_BASM_TXIDS = previousLimit + } + } + }) + it('enforces admin authentication and executes bounded record and ban operations', async () => { instance.configureAdminIdentityKey('admin-identity') const banService = { diff --git a/packages/overlays/overlay/CHANGELOG.md b/packages/overlays/overlay/CHANGELOG.md index 8bdf170c4..bf11cf34f 100644 --- a/packages/overlays/overlay/CHANGELOG.md +++ b/packages/overlays/overlay/CHANGELOG.md @@ -20,6 +20,13 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +- Added bounded BASM JSON peer validation, classified capability/resource errors, + canonical header and optional full-block-count position checks, and explicit + position assurance in sync reports. Fixed default forward pages to fit the + public server's 1,000-anchor limit. Historical sync refuses an untrusted TAC + prefix; durable bootstrap, chunked recovery and topic status remain pending. + No storage migration or automatic sync activation is included. + ### Added - (Include new features or significant user-visible enhancements here.) diff --git a/packages/overlays/overlay/README.md b/packages/overlays/overlay/README.md index c8c87fab5..540110cc7 100644 --- a/packages/overlays/overlay/README.md +++ b/packages/overlays/overlay/README.md @@ -120,6 +120,51 @@ For production deployments: `@bsv/overlay-express` supplies these standard HTTP controls while preserving public protocol access by default. +### BASM peer validation and current recovery limits + +BASM uses the current BRC-136 ordered admitted subset and block-anchored TAC. +The five existing JSON POST routes remain compatible; empty tips remain +`{ topic, blockHeight: -1, tac: }`. Unsupported storage capabilities +are errors, not empty histories. The remote client validates response shape, +topic/height/hash binding, ordered unique admitted positions, contiguous +returned ranges, and complete proof/raw response ID sets before use. + +`BASMRemote` retains its injectable third `fetch` argument and accepts optional +limits as a fourth argument. Defaults are 64 MiB per decoded response, 8 MiB +per proof, 32 MiB per raw transaction, 100,000 admitted entries, 1,000 requested +txids, 1,024 requested anchor heights, and 30 seconds per request including its +body. Aggregate response limits also apply to hex-encoded transactions. These +are configurable local acceptance limits, not consensus rules. Standard fetch +bodies are bounded while streaming; legacy injected `text()` implementations +are checked after buffering. Classified errors expose `code`, including +`BASM_UNSUPPORTED`, `BASM_RESOURCE_LIMIT`, and `BASM_TIMEOUT`. + +Reconciliation requires a canonical header resolver as well as a ChainTracker. +An optional `TopicAnchorHeader.blockTransactionCount` must come independently +from the trusted canonical provider and refer to that exact `blockHash`. +It enables full-block count/index bounds and odd-duplication checks. The sync +report's `positionValidation` is `canonical-count` only when that evidence was +available for every checked proof; legacy providers yield `encoded-offset-only`. +A Merkle root plus an encoded offset alone cannot disambiguate Bitcoin's +duplicate-last-leaf position ambiguity. No provider is required to add the +field, and the engine does not download full blocks to infer it. + +Forward sync pages now contain at most 1,000 anchors to fit the standard HTTP +server. Proof height, requested original index, canonical hash/root, raw byte +identity, TAC continuity, and repeated peer anchors are checked before historical +submission. Historical mode still applies the local TopicManager and suppresses +broadcast and propagation. Automatic BASM sync remains disabled by default. + +This is bounded protocol hardening, not durable recovery. An empty local node +whose topic genesis precedes the recent bootstrap window now refuses the +untrusted TAC prefix; this intentionally replaces the old unchecked tail +behavior. A block above 1,000 admitted entries reaches a request-limit error +until proof/raw chunking is implemented. Equal-height/local-ahead divergence, +whole-target bootstrap, durable cursors/leases, atomic revision fencing, and +truthful per-topic agreement status remain required follow-up work. A successful +legacy report does not establish global completeness, current unspentness, or +durable recovery completion. See [BASM details](./docs/BRC-136-BASM.md). + ## Development From the repository root: diff --git a/packages/overlays/overlay/docs/BRC-136-BASM.md b/packages/overlays/overlay/docs/BRC-136-BASM.md index 4cd1ed46d..59e17bd25 100644 --- a/packages/overlays/overlay/docs/BRC-136-BASM.md +++ b/packages/overlays/overlay/docs/BRC-136-BASM.md @@ -7,6 +7,14 @@ topic**. The two systems are deliberately decoupled. Read the final section ("How BASM and removal interact") before operating a node that runs the janitor or removes tokens via the admin API. +The protocol's binary-search reconciliation is a design capability, not an +implemented durable recovery scheduler in this release. The current engine +performs bounded forward work and refuses historical divergence or an untrusted +bootstrap prefix. See the [current validation and recovery limits](../README.md#basm-peer-validation-and-current-recovery-limits) +before interpreting a sync report. Optional canonical full-block counts provide +stronger original-position evidence; header-only providers retain explicitly +weaker evidence. Peer anchor equality is not durable or global completeness. + ## What BRC-136 gives you GASP ([Synchronization.md](./Synchronization.md)) replicates transaction *data* diff --git a/packages/overlays/overlay/package.json b/packages/overlays/overlay/package.json index 7f518ae4a..f1be11b58 100644 --- a/packages/overlays/overlay/package.json +++ b/packages/overlays/overlay/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay", - "version": "2.3.1", + "version": "2.4.0", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/overlay/src/BASM.ts b/packages/overlays/overlay/src/BASM.ts index 470894d8b..3ec4d61fa 100644 --- a/packages/overlays/overlay/src/BASM.ts +++ b/packages/overlays/overlay/src/BASM.ts @@ -44,9 +44,13 @@ export interface TopicAnchorHeader { blockHeight: number blockHash: string merkleRoot?: string + /** Independently obtained full block transaction count, bound to this blockHash. */ + blockTransactionCount?: number } -export type TopicAnchorHeaderResolver = (blockHeight: number) => Promise +export type TopicAnchorHeaderResolver = ( + blockHeight: number +) => Promise export interface TopicAnchorRangeRequest { fromHeight: number @@ -106,6 +110,10 @@ export interface BASMPeerSyncReport { checkedHeights: number[] missingTxids: string[] fetchedTxCount: number + /** Weakest position evidence used by this attempt; absent when no proof was checked. */ + positionValidation?: 'canonical-count' | 'encoded-offset-only' + /** Present for classified peer protocol or transport failures. */ + errorCode?: string message?: string } @@ -190,7 +198,10 @@ export function computeTac(prevTac: string, blockHash: string, basmRoot: string) return internalToDisplayHex(sha256d(input)) } -export function extractMerkleProofMetadata(txid: string, proof?: MerklePath): MerkleProofMetadata | undefined { +export function extractMerkleProofMetadata( + txid: string, + proof?: MerklePath +): MerkleProofMetadata | undefined { if (proof === undefined) { return undefined } diff --git a/packages/overlays/overlay/src/BASMRemote.ts b/packages/overlays/overlay/src/BASMRemote.ts index d30a58807..d426520ca 100644 --- a/packages/overlays/overlay/src/BASMRemote.ts +++ b/packages/overlays/overlay/src/BASMRemote.ts @@ -5,35 +5,230 @@ import type { TopicAnchorRangeResponse, TopicAnchorTip } from './BASM.js' +import { + BASMProtocolError, + DEFAULT_BASM_REMOTE_LIMITS, + basmAdmitted, + basmAnchor, + basmHash, + basmHex, + basmInteger, + basmObject, + basmTip, + basmTxids, + requireBASM, + requireBASMLimit +} from './BASMValidation.js' +import type { BASMRemoteLimits } from './BASMValidation.js' + +export { BASMProtocolError } from './BASMValidation.js' +export type { BASMRemoteLimits } from './BASMValidation.js' export class BASMRemote { + private readonly limits: Readonly + constructor( private readonly endpoint: string, private readonly topic: string, - private readonly fetchImpl: typeof fetch = fetch.bind(globalThis) - ) { } + private readonly fetchImpl: typeof fetch = fetch.bind(globalThis), + limits: Partial = {} + ) { + this.limits = Object.freeze({ ...DEFAULT_BASM_REMOTE_LIMITS, ...limits }) + for (const [key, value] of Object.entries(this.limits)) { + if (!Number.isSafeInteger(value) || value <= 0) + throw new TypeError(`Invalid BASM limit: ${key}`) + } + if (this.limits.timeoutMs > 2147483647) throw new TypeError('BASM timeout exceeds timer range') + } async requestTopicAnchorTip(): Promise { - return await this.post('/requestTopicAnchorTip', {}) + return basmTip(await this.post('/requestTopicAnchorTip', {}), this.topic) } - async requestTopicAnchorRange(fromHeight: number, toHeight: number): Promise { - return await this.post('/requestTopicAnchorRange', { fromHeight, toHeight }) + async requestTopicAnchorRange( + fromHeight: number, + toHeight: number + ): Promise { + basmInteger(fromHeight, 'range start') + basmInteger(toHeight, 'range end', fromHeight) + requireBASMLimit( + toHeight - fromHeight < this.limits.maxAnchorRange, + 'BASM anchor range exceeds limit' + ) + const obj = basmObject(await this.post('/requestTopicAnchorRange', { fromHeight, toHeight })) + requireBASM(obj.topic === this.topic && Array.isArray(obj.anchors), 'Invalid BASM anchor range') + requireBASMLimit( + obj.anchors.length <= toHeight - fromHeight + 1, + 'BASM anchor count exceeds requested range' + ) + const anchors = obj.anchors.map(value => basmAnchor(value, this.topic)) + let previousHeight: number | undefined + for (const anchor of anchors) { + requireBASM( + anchor.blockHeight >= fromHeight && anchor.blockHeight <= toHeight, + 'BASM anchor outside requested range' + ) + requireBASM( + previousHeight === undefined || anchor.blockHeight === previousHeight + 1, + 'BASM anchor range has a gap or is unordered' + ) + previousHeight = anchor.blockHeight + } + return { topic: this.topic, anchors } } - async requestAdmittedList(blockHeight: number, blockHash?: string): Promise { - return await this.post('/requestAdmittedList', { blockHeight, blockHash }) + async requestAdmittedList( + blockHeight: number, + blockHash?: string + ): Promise { + basmInteger(blockHeight, 'height') + if (blockHash !== undefined) basmHash(blockHash, 'block hash') + const obj = basmObject(await this.post('/requestAdmittedList', { blockHeight, blockHash })) + this.requireCoordinate(obj, blockHeight) + const responseHash = + obj.blockHash === undefined ? undefined : basmHash(obj.blockHash, 'block hash') + requireBASM( + blockHash === undefined || responseHash === undefined || responseHash === blockHash, + 'BASM admitted block hash mismatch' + ) + return { + topic: this.topic, + blockHeight, + ...(responseHash === undefined ? {} : { blockHash: responseHash }), + admitted: basmAdmitted(obj.admitted, this.limits.maxAdmittedTxids) + } } - async requestCompoundMerklePath(blockHeight: number, txids: string[]): Promise { - return await this.post('/requestCompoundMerklePath', { blockHeight, txids }) + async requestCompoundMerklePath( + blockHeight: number, + txids: string[] + ): Promise { + basmInteger(blockHeight, 'height') + const requested = basmTxids(txids, this.limits.maxRequestedTxids) + requireBASM(requested.length > 0, 'At least one BASM proof txid is required') + const obj = basmObject( + await this.post('/requestCompoundMerklePath', { blockHeight, txids: requested }) + ) + this.requireCoordinate(obj, blockHeight) + const returned = basmTxids(obj.txids, requested.length) + const requestedSet = new Set(requested) + requireBASM( + returned.length === requested.length && returned.every(txid => requestedSet.has(txid)), + 'BASM proof txids do not match request' + ) + return { + topic: this.topic, + blockHeight, + txids: returned, + merklePath: basmHex(obj.merklePath, 'Merkle path', this.limits.maxProofBytes) + } } async requestRawTransactions(txids: string[]): Promise { - return await this.post('/requestRawTransactions', { txids }) + const requested = new Set(basmTxids(txids, this.limits.maxRequestedTxids)) + const obj = basmObject(await this.post('/requestRawTransactions', { txids: [...requested] })) + requireBASM(Array.isArray(obj.transactions), 'Invalid BASM raw transactions') + requireBASMLimit( + obj.transactions.length <= requested.size, + 'BASM raw transaction count exceeds request' + ) + const seen = new Set() + const transactions = obj.transactions.map(value => { + const record = basmObject(value) + const txid = basmHash(record.txid, 'txid') + requireBASM( + requested.has(txid) && !seen.has(txid), + 'Unexpected or duplicate BASM raw transaction' + ) + seen.add(txid) + return { + txid, + rawTx: basmHex(record.rawTx, 'raw transaction', this.limits.maxRawTransactionBytes) + } + }) + const missing = basmTxids(obj.missing, requested.size) + for (const txid of missing) { + requireBASM( + requested.has(txid) && !seen.has(txid), + 'Unexpected or conflicting BASM missing txid' + ) + seen.add(txid) + } + requireBASM(seen.size === requested.size, 'BASM raw response omits requested txids') + return { transactions, missing } + } + + private requireCoordinate(obj: Record, blockHeight: number): void { + requireBASM( + obj.topic === this.topic && obj.blockHeight === blockHeight, + 'BASM response topic or height mismatch' + ) } - private async post(path: string, body: unknown): Promise { + private async readResponse(response: Response, signal: AbortSignal): Promise { + const advertisedLength = response.headers?.get('content-length') + if (advertisedLength !== null && advertisedLength !== undefined) { + if (!(Number(advertisedLength) <= this.limits.maxResponseBytes)) { + void response.body?.cancel().catch(() => {}) + requireBASMLimit(false, 'BASM response exceeds byte limit') + } + } + // Preserve injected fetch implementations that expose only text(). Real HTTP + // responses are bounded while streaming, including decoded chunked bodies. + if (response.body === undefined || response.body === null) { + const text = await response.text() + requireBASMLimit( + Buffer.byteLength(text) <= this.limits.maxResponseBytes, + 'BASM response exceeds byte limit' + ) + return text + } + const reader = response.body.getReader() + const chunks: Uint8Array[] = [] + let size = 0 + const cancel = (): void => { + void reader.cancel().catch(() => {}) + } + signal.addEventListener('abort', cancel, { once: true }) + try { + while (true) { + signal.throwIfAborted() + const { done, value } = await reader.read() + if (done) break + size += value.byteLength + requireBASMLimit(size <= this.limits.maxResponseBytes, 'BASM response exceeds byte limit') + chunks.push(value) + } + signal.throwIfAborted() + return Buffer.concat(chunks, size).toString('utf8') + } catch (error) { + cancel() + throw error + } finally { + signal.removeEventListener('abort', cancel) + reader.releaseLock() + } + } + + private async post(path: string, body: unknown): Promise { + const controller = new AbortController() + let timer: ReturnType | undefined + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + const error = new BASMProtocolError('BASM_TIMEOUT', 'BASM request timed out') + controller.abort(error) + reject(error) + }, this.limits.timeoutMs) + }) + try { + return await Promise.race([this.postResponse(path, body, controller.signal), deadline]) + } finally { + clearTimeout(timer) + controller.abort() + } + } + + private async postResponse(path: string, body: unknown, signal: AbortSignal): Promise { const response = await this.fetchImpl(new URL(path, this.endpoint).toString(), { method: 'POST', headers: { @@ -41,14 +236,32 @@ export class BASMRemote { 'Content-Type': 'application/json', 'x-bsv-topic': this.topic }, - body: JSON.stringify(body) + body: JSON.stringify(body), + signal }) - - const text = await response.text() + if (signal.aborted) { + void response.body?.cancel().catch(() => {}) + signal.throwIfAborted() + } + const text = await this.readResponse(response, signal) + let value: unknown + try { + value = JSON.parse(text) + } catch { + if (response.ok) + throw new BASMProtocolError('BASM_INVALID_RESPONSE', 'Invalid BASM JSON response') + } if (!response.ok) { - throw new Error(`BASM peer ${this.endpoint} returned ${response.status}: ${text}`) + const code = + typeof value === 'object' && value !== null && 'code' in value ? value.code : undefined + if (response.status === 501 || code === 'BASM_UNSUPPORTED') { + throw new BASMProtocolError( + 'BASM_UNSUPPORTED', + `BASM peer returned HTTP ${response.status}: unsupported BASM capability` + ) + } + throw new BASMProtocolError('BASM_HTTP_ERROR', `BASM peer returned HTTP ${response.status}`) } - - return (text.length === 0 ? {} : JSON.parse(text)) as T + return value } } diff --git a/packages/overlays/overlay/src/BASMValidation.ts b/packages/overlays/overlay/src/BASMValidation.ts new file mode 100644 index 000000000..fa6721cb2 --- /dev/null +++ b/packages/overlays/overlay/src/BASMValidation.ts @@ -0,0 +1,131 @@ +import { BASM_ZERO_HASH } from './BASM.js' +import type { AdmittedTxRef, TopicAnchorTip, TopicBlockAnchor } from './BASM.js' + +/** Local acceptance limits, not Bitcoin consensus limits. */ +export interface BASMRemoteLimits { + maxResponseBytes: number + maxProofBytes: number + maxRawTransactionBytes: number + maxAdmittedTxids: number + maxRequestedTxids: number + maxAnchorRange: number + timeoutMs: number +} + +export const DEFAULT_BASM_REMOTE_LIMITS: Readonly = Object.freeze({ + maxResponseBytes: 64 * 1024 * 1024, + maxProofBytes: 8 * 1024 * 1024, + maxRawTransactionBytes: 32 * 1024 * 1024, + maxAdmittedTxids: 100000, + maxRequestedTxids: 1000, + maxAnchorRange: 1024, + timeoutMs: 30000 +}) + +export class BASMProtocolError extends Error { + constructor( + public readonly code: + | 'BASM_INVALID_RESPONSE' + | 'BASM_RESOURCE_LIMIT' + | 'BASM_UNSUPPORTED' + | 'BASM_TIMEOUT' + | 'BASM_HTTP_ERROR', + message: string + ) { + super(message) + this.name = 'BASMProtocolError' + } +} + +export function requireBASM(condition: boolean, message: string): asserts condition { + if (!condition) throw new BASMProtocolError('BASM_INVALID_RESPONSE', message) +} + +export function requireBASMLimit(condition: boolean, message: string): void { + if (!condition) throw new BASMProtocolError('BASM_RESOURCE_LIMIT', message) +} + +export function basmObject(value: unknown): Record { + requireBASM( + typeof value === 'object' && value !== null && !Array.isArray(value), + 'Expected a BASM JSON object' + ) + return value as Record +} + +export function basmInteger(value: unknown, label: string, minimum = 0): number { + requireBASM( + typeof value === 'number' && Number.isSafeInteger(value) && value >= minimum, + `Invalid BASM ${label}` + ) + return value +} + +export function basmHash(value: unknown, label: string): string { + requireBASM(typeof value === 'string' && /^[0-9a-f]{64}$/.test(value), `Invalid BASM ${label}`) + return value +} + +export function basmHex(value: unknown, label: string, maxBytes: number): string { + requireBASM(typeof value === 'string', `Invalid BASM ${label}`) + requireBASMLimit(value.length <= maxBytes * 2, `BASM ${label} exceeds byte limit`) + requireBASM( + value.length > 0 && value.length % 2 === 0 && /^[0-9a-fA-F]+$/.test(value), + `Invalid BASM ${label}` + ) + return value +} + +export function basmTxids(value: unknown, maximum: number): string[] { + requireBASM(Array.isArray(value), 'Invalid BASM txids') + requireBASMLimit(value.length <= maximum, 'BASM txid count exceeds limit') + const txids = value.map(txid => basmHash(txid, 'txid')) + requireBASM(new Set(txids).size === txids.length, 'Duplicate BASM txid') + return txids +} + +export function basmTip(value: unknown, topic: string): TopicAnchorTip { + const obj = basmObject(value) + requireBASM(obj.topic === topic, 'BASM topic mismatch') + const blockHeight = basmInteger(obj.blockHeight, 'tip height', -1) + const tac = basmHash(obj.tac, 'TAC') + const tip: TopicAnchorTip = { topic, blockHeight, tac } + if (obj.blockHash !== undefined) tip.blockHash = basmHash(obj.blockHash, 'block hash') + if (obj.basmRoot !== undefined) tip.basmRoot = basmHash(obj.basmRoot, 'root') + if (obj.admittedCount !== undefined) + tip.admittedCount = basmInteger(obj.admittedCount, 'admitted count') + if (blockHeight === -1) requireBASM(tac === BASM_ZERO_HASH, 'Empty BASM tip must have zero TAC') + return tip +} + +export function basmAnchor(value: unknown, topic: string): TopicBlockAnchor { + const obj = basmObject(value) + const tip = basmTip(value, topic) + return { + topic, + blockHeight: basmInteger(tip.blockHeight, 'anchor height'), + blockHash: basmHash(obj.blockHash, 'block hash'), + basmRoot: basmHash(obj.basmRoot, 'root'), + admittedCount: basmInteger(obj.admittedCount, 'admitted count'), + tac: tip.tac + } +} + +export function basmAdmitted(value: unknown, maximum: number): AdmittedTxRef[] { + requireBASM(Array.isArray(value), 'Invalid BASM admitted list') + requireBASMLimit(value.length <= maximum, 'BASM admitted count exceeds limit') + let previousIndex = -1 + const seen = new Set() + return value.map(item => { + const obj = basmObject(item) + const txid = basmHash(obj.txid, 'txid') + const blockIndex = basmInteger(obj.blockIndex, 'block index') + requireBASM( + blockIndex > previousIndex && !seen.has(txid), + 'BASM admitted list must have unique txids in increasing block order' + ) + previousIndex = blockIndex + seen.add(txid) + return { txid, blockIndex } + }) +} diff --git a/packages/overlays/overlay/src/Engine.ts b/packages/overlays/overlay/src/Engine.ts index 5022b010d..1bc434650 100644 --- a/packages/overlays/overlay/src/Engine.ts +++ b/packages/overlays/overlay/src/Engine.ts @@ -27,11 +27,13 @@ import { OverlayGASPRemote } from './GASP/OverlayGASPRemote.js' import { OverlayGASPStorage } from './GASP/OverlayGASPStorage.js' import { BASM_ZERO_HASH, + type AdmittedTxRef, type AdmittedListResponse, type BASMPeerSyncReport, type CompoundMerklePathResponse, type RawTransactionResponse, type ReorgReport, + type TopicAnchorHeader, type TopicAnchorHeaderResolver, type TopicAnchorRangeResponse, type TopicAnchorTip, @@ -41,10 +43,13 @@ import { extractMerkleProofMetadata } from './BASM.js' import { BASMRemote } from './BASMRemote.js' +import { basmHash, basmInteger, requireBASM } from './BASMValidation.js' import { serializeErrorForLog, serializeLogValue } from './SafeLog.js' const DEFAULT_GASP_SYNC_LIMIT = 10000 const DEFAULT_BASM_RANGE_LIMIT = 1024 +// The public Overlay Express transport defaults to 1,000 anchors per request. +const DEFAULT_BASM_SYNC_PAGE_SIZE = 1000 type UTXOHistoryHydrationContext = { outputCache: Map> @@ -1541,6 +1546,9 @@ export class Engine { } async provideTopicAnchorTip(topic: string): Promise { + if (typeof this.storage.findTopicAnchorTip !== 'function') { + throw Object.assign(new TypeError('Storage does not support BASM topic anchor tips'), { code: 'BASM_UNSUPPORTED' }) + } const tip = await this.storage.findTopicAnchorTip?.(topic) return tip ?? { topic, @@ -1551,7 +1559,7 @@ export class Engine { async provideTopicAnchorRange(topic: string, fromHeight: number, toHeight: number): Promise { if (typeof this.storage.findTopicBlockAnchors !== 'function') { - throw new TypeError('Storage does not support BASM topic anchor ranges') + throw Object.assign(new TypeError('Storage does not support BASM topic anchor ranges'), { code: 'BASM_UNSUPPORTED' }) } if (!Number.isInteger(fromHeight) || !Number.isInteger(toHeight) || fromHeight < 0 || toHeight < fromHeight) { throw new Error('Invalid topic anchor range') @@ -1568,7 +1576,7 @@ export class Engine { async provideAdmittedList(topic: string, blockHeight: number, blockHash?: string): Promise { if (typeof this.storage.findAdmittedTransactionsForBlock !== 'function') { - throw new TypeError('Storage does not support BASM admitted lists') + throw Object.assign(new TypeError('Storage does not support BASM admitted lists'), { code: 'BASM_UNSUPPORTED' }) } return { @@ -1581,7 +1589,7 @@ export class Engine { async provideCompoundMerklePath(topic: string, blockHeight: number, txids: string[]): Promise { if (typeof this.storage.findTransactionMerklePaths !== 'function') { - throw new TypeError('Storage does not support direct Merkle path lookup') + throw Object.assign(new TypeError('Storage does not support direct Merkle path lookup'), { code: 'BASM_UNSUPPORTED' }) } if (txids.length === 0) { throw new Error('At least one txid is required') @@ -1632,7 +1640,7 @@ export class Engine { async provideRawTransactions(txids: string[]): Promise { if (typeof this.storage.findRawTransactions !== 'function') { - throw new TypeError('Storage does not support raw transaction lookup') + throw Object.assign(new TypeError('Storage does not support raw transaction lookup'), { code: 'BASM_UNSUPPORTED' }) } const transactions = await this.storage.findRawTransactions(txids) @@ -1678,7 +1686,22 @@ export class Engine { report.localTip = localTip report.remoteTip = remoteTip + if (remoteTip.blockHeight >= 0) { + const tipRange = await remote.requestTopicAnchorRange(remoteTip.blockHeight, remoteTip.blockHeight) + const tipAnchor = tipRange.anchors[0] + requireBASM(tipAnchor !== undefined && tipAnchor.tac === remoteTip.tac, 'BASM tip does not match its anchor') + for (const field of ['blockHash', 'basmRoot', 'admittedCount'] as const) { + requireBASM(remoteTip[field] === undefined || remoteTip[field] === tipAnchor[field], 'BASM tip metadata does not match its anchor') + } + await this.requireCanonicalBASMAnchor(tipAnchor) + } + if (localTip.blockHeight >= remoteTip.blockHeight) { + if (localTip.blockHeight >= 0 && localTip.blockHeight === remoteTip.blockHeight && localTip.tac === remoteTip.tac) { + const localAnchor = await this.storage.findTopicBlockAnchor?.(topic, localTip.blockHeight) + requireBASM(localAnchor !== undefined && localAnchor.tac === localTip.tac, 'Local BASM tip lacks its anchor') + await this.requireCanonicalBASMAnchor(localAnchor) + } report.status = localTip.tac === remoteTip.tac && localTip.blockHeight === remoteTip.blockHeight ? 'matched' : 'diverged' report.message = report.status === 'matched' ? 'Topic anchor tips match' @@ -1686,30 +1709,78 @@ export class Engine { return report } - const fromHeight = Math.max(localTip.blockHeight + 1, remoteTip.blockHeight - DEFAULT_BASM_RANGE_LIMIT + 1, 0) - const range = await remote.requestTopicAnchorRange(fromHeight, remoteTip.blockHeight) + const fromHeight = localTip.blockHeight < 0 + ? Math.max(remoteTip.blockHeight - DEFAULT_BASM_SYNC_PAGE_SIZE + 1, 0) + : localTip.blockHeight + 1 + const toHeight = Math.min(fromHeight + DEFAULT_BASM_SYNC_PAGE_SIZE - 1, remoteTip.blockHeight) + const range = await remote.requestTopicAnchorRange(fromHeight, toHeight) + requireBASM(range.anchors.length > 0 && range.anchors.at(-1)?.blockHeight === toHeight, 'BASM range omits its requested target') + requireBASM(localTip.blockHeight < 0 || range.anchors[0].blockHeight === fromHeight, 'BASM range omits its next height') + let previousTac = localTip.tac + for (const anchor of range.anchors) { + requireBASM(anchor.tac === computeTac(previousTac, anchor.blockHash, anchor.basmRoot), 'BASM range TAC is inconsistent with its prefix') + previousTac = anchor.tac + } + if (toHeight === remoteTip.blockHeight) requireBASM(previousTac === remoteTip.tac, 'BASM range differs from its tip') for (const remoteAnchor of range.anchors) { await this.reconcileRemoteAnchor(topic, remote, remoteAnchor, report) } + const finalRemoteTip = await remote.requestTopicAnchorTip() + requireBASM(finalRemoteTip.blockHeight === remoteTip.blockHeight && finalRemoteTip.tac === remoteTip.tac, 'BASM peer history changed during reconciliation') + const refreshedTip = await this.provideTopicAnchorTip(topic) report.localTip = refreshedTip - report.status = refreshedTip.blockHeight >= remoteTip.blockHeight && refreshedTip.tac === remoteTip.tac ? 'matched' : 'advanced' + report.status = refreshedTip.blockHeight === remoteTip.blockHeight && refreshedTip.tac === remoteTip.tac ? 'matched' : 'advanced' return report } catch (error) { report.status = 'error' + if (error instanceof Error && 'code' in error && typeof error.code === 'string') report.errorCode = error.code report.message = error instanceof Error ? error.message : String(error) this.logger.error(`[BASM SYNC] Sync failed for topic "${topic}" with peer "${endpoint}"`, error) return report } } + private async requireCanonicalBASMAnchor(anchor: TopicBlockAnchor, proofRoot?: string): Promise { + if (this.chainTracker === 'scripts only' || this.topicAnchorHeaderResolver === undefined) { + throw new Error('BASM reconciliation requires a ChainTracker and canonical header resolver') + } + const header = await this.topicAnchorHeaderResolver(anchor.blockHeight) + requireBASM(header !== undefined && header.blockHeight === anchor.blockHeight, 'BASM canonical header is unavailable or has the wrong height') + requireBASM(basmHash(header.blockHash.toLowerCase(), 'canonical block hash') === anchor.blockHash, 'BASM anchor block hash is not canonical') + if (proofRoot !== undefined && header.merkleRoot !== undefined) { + requireBASM(header.merkleRoot.toLowerCase() === proofRoot, 'BASM proof root differs from its canonical header') + } + if (header.blockTransactionCount !== undefined) { + basmInteger(header.blockTransactionCount, 'canonical block transaction count', 1) + requireBASM(anchor.admittedCount <= header.blockTransactionCount, 'BASM admitted count exceeds canonical block transaction count') + } + return header + } + + private validateBASMProofPositions(path: MerklePath, admitted: AdmittedTxRef[], count: number): void { + requireBASM(admitted.every(item => item.blockIndex < count), 'BASM admitted index exceeds canonical block transaction count') + let width = count + for (let height = 0; height < path.path.length; height++) { + requireBASM(height === 0 || width > 1, 'BASM proof exceeds canonical tree depth') + for (const node of path.path[height]) { + requireBASM( + node.duplicate === true ? width % 2 === 1 && node.offset === width : node.offset < width, + 'BASM proof node is outside canonical block positions' + ) + } + width = Math.ceil(width / 2) + } + } + private async reconcileRemoteAnchor( topic: string, remote: BASMRemote, remoteAnchor: TopicBlockAnchor, report: BASMPeerSyncReport ): Promise { + await this.requireCanonicalBASMAnchor(remoteAnchor) report.checkedHeights.push(remoteAnchor.blockHeight) const localAnchor = await this.storage.findTopicBlockAnchor?.(topic, remoteAnchor.blockHeight, remoteAnchor.blockHash) if (localAnchor?.tac === remoteAnchor.tac) { @@ -1737,7 +1808,8 @@ export class Engine { return } - await this.fetchBASMMissingTransactions(remote, topic, remoteAnchor, missingTxids) + const assurance = await this.fetchBASMMissingTransactions(remote, topic, remoteAnchor, admittedResponse.admitted, missingTxids) + if (report.positionValidation !== 'encoded-offset-only') report.positionValidation = assurance report.fetchedTxCount += missingTxids.length } @@ -1745,15 +1817,28 @@ export class Engine { remote: BASMRemote, topic: string, anchor: TopicBlockAnchor, + admitted: AdmittedTxRef[], txids: string[] - ): Promise { + ): Promise<'canonical-count' | 'encoded-offset-only'> { if (this.chainTracker === 'scripts only') { throw new Error('BASM reconciliation requires a ChainTracker capable of validating BUMP proofs') } - const proofResponse = await remote.requestCompoundMerklePath(anchor.blockHeight, txids) + // Validate the whole claimed ordered subset, including entries already local: + // the BASM root alone does not bind the peer's claimed original positions. + const proofResponse = await remote.requestCompoundMerklePath(anchor.blockHeight, admitted.map(item => item.txid)) const compoundPath = MerklePath.fromHex(proofResponse.merklePath) - for (const txid of txids) { + requireBASM(compoundPath.blockHeight === anchor.blockHeight, 'BASM proof height does not match its anchor') + requireBASM(compoundPath.toHex() === proofResponse.merklePath.toLowerCase(), 'BASM proof is not canonically encoded') + const proofRoot = compoundPath.computeRoot() + const proofHeader = await this.requireCanonicalBASMAnchor(anchor, proofRoot) + if (proofHeader.blockTransactionCount !== undefined) { + this.validateBASMProofPositions(compoundPath, admitted, proofHeader.blockTransactionCount) + } + for (const { txid, blockIndex } of admitted) { + const leaf = compoundPath.path[0]?.find(item => item.hash === txid) + requireBASM(leaf !== undefined && leaf.offset === blockIndex, 'BASM proof does not bind the admitted block index') + requireBASM(compoundPath.path[0].length !== 1 || compoundPath.path.length !== 1 || blockIndex === 0, 'BASM singleton proof has a nonzero block index') const valid = await compoundPath.verify(txid, this.chainTracker) if (!valid) { throw new Error(`Peer supplied invalid compound Merkle path for ${txid} at height ${anchor.blockHeight}`) @@ -1765,18 +1850,33 @@ export class Engine { throw new Error(`Peer did not return raw transactions for txids: ${rawResponse.missing.join(',')}`) } - for (const record of rawResponse.transactions) { + const transactions = rawResponse.transactions.map(record => { const tx = Transaction.fromHex(record.rawTx) - if (tx.id('hex') !== record.txid) { + if (tx.id('hex') !== record.txid || tx.toHex() !== record.rawTx.toLowerCase()) { throw new Error(`Raw transaction txid mismatch: expected ${record.txid}, got ${tx.id('hex')}`) } - try { - tx.merklePath = compoundPath.extract([record.txid]) - } catch { - tx.merklePath = compoundPath - } + tx.merklePath = compoundPath.extract([record.txid]) + return tx + }) + const refreshedAnchor = (await remote.requestTopicAnchorRange(anchor.blockHeight, anchor.blockHeight)).anchors[0] + requireBASM( + refreshedAnchor !== undefined && refreshedAnchor.blockHash === anchor.blockHash && + refreshedAnchor.basmRoot === anchor.basmRoot && refreshedAnchor.admittedCount === anchor.admittedCount && + refreshedAnchor.tac === anchor.tac, + 'BASM peer anchor changed before admission' + ) + const commitHeader = await this.requireCanonicalBASMAnchor(anchor, proofRoot) + requireBASM(commitHeader.blockTransactionCount === proofHeader.blockTransactionCount, 'BASM canonical block transaction count changed before admission') + // Apply in the independently checked block order, regardless of raw response order. + const transactionById = new Map(transactions.map(tx => [tx.id('hex'), tx])) + for (const txid of txids) { + const tx = transactionById.get(txid) + requireBASM(tx !== undefined, 'BASM raw response omits a requested transaction') await this.submit({ beef: tx.toBEEF(), topics: [topic] }, undefined, 'historical-tx') } + const finalHeader = await this.requireCanonicalBASMAnchor(anchor, proofRoot) + requireBASM(finalHeader.blockTransactionCount === proofHeader.blockTransactionCount, 'BASM canonical block transaction count changed during admission') + return proofHeader.blockTransactionCount === undefined ? 'encoded-offset-only' : 'canonical-count' } async evictUnprovenTransactions(options: { diff --git a/packages/overlays/overlay/src/__tests/BASM.conformance.test.ts b/packages/overlays/overlay/src/__tests/BASM.conformance.test.ts new file mode 100644 index 000000000..3117517a5 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/BASM.conformance.test.ts @@ -0,0 +1,165 @@ +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { computeBasmRoot, computeTac, BASM_ZERO_HASH } from '../BASM' +import { basmAdmitted } from '../BASMValidation' + +interface MerkleVector { + name: string + txids: string[] + root: string + admissionListValid: boolean +} + +interface TacAnchor { + blockHeight: number + blockHash: string + basmRoot: string + admittedCount: number + rootSource: string + expectedTac: string +} + +interface TacVector { + name: string + genesisHeight: number + anchors: TacAnchor[] +} + +interface ByteOrderVector { + name: string + display: string + internal: string +} + +const FIXTURE_PATH = join(__dirname, 'fixtures', 'brc136-independent.json') +const FIXTURE_SHA256 = '51983432bb561e3031fb7c983947dc8bfdcfd0ef828ef9f24093666ff4665c8a' +const fixtureBytes = readFileSync(FIXTURE_PATH) +const fixture = JSON.parse(fixtureBytes.toString('utf8')) as { + specRevision: { brc136: string; repoHead: string } + hashEncoding: string + byteOrder: ByteOrderVector[] + merkle: MerkleVector[] + tac: TacVector[] +} +const merkleVectors = fixture.merkle +const tacVectors = fixture.tac + +function displayToInternal(hash: string): Buffer { + if (!/^[0-9a-f]{64}$/.test(hash)) { + throw new Error(`expected lowercase 32-byte hex, got ${hash}`) + } + return Buffer.from(hash, 'hex').reverse() +} + +function internalToDisplay(value: Buffer): string { + if (value.length !== 32) throw new Error('expected 32 bytes') + return Buffer.from(value).reverse().toString('hex') +} + +function sha256d(value: Buffer): Buffer { + return createHash('sha256').update(createHash('sha256').update(value).digest()).digest() +} + +function opensslSha256d(value: Buffer): Buffer { + const first = execFileSync('openssl', ['dgst', '-sha256', '-binary'], { input: value }) + return execFileSync('openssl', ['dgst', '-sha256', '-binary'], { input: first }) +} + +function independentBasmRoot(txids: string[], digest: (value: Buffer) => Buffer): string { + if (txids.length === 0) return BASM_ZERO_HASH + let layer = txids.map(displayToInternal) + while (layer.length > 1) { + const next: Buffer[] = [] + for (let index = 0; index < layer.length; index += 2) { + const left = layer[index] + const right = index + 1 < layer.length ? layer[index + 1] : left + next.push(digest(Buffer.concat([left, right]))) + } + layer = next + } + return internalToDisplay(layer[0]) +} + +function independentTac( + previous: string, + blockHash: string, + root: string, + digest: (value: Buffer) => Buffer +): string { + return internalToDisplay( + digest( + Buffer.concat([ + displayToInternal(previous), + displayToInternal(blockHash), + displayToInternal(root) + ]) + ) + ) +} + +const opensslAvailable = ((): boolean => { + try { + execFileSync('openssl', ['version'], { stdio: 'ignore' }) + return true + } catch { + return false + } +})() + +describe('independent BRC-136 conformance vectors', () => { + it('pins the BRC revision and frozen fixture bytes', () => { + expect(fixture.specRevision.brc136).toBe('2733cd2950a739b3c977b95d652ff63e3773c40b') + expect(fixture.specRevision.repoHead).toBe('39a643ff148a8dcd23ec08986a8ddeb7d5713743') + expect(fixture.hashEncoding).toBe('lowercase display-order hex') + expect(createHash('sha256').update(fixtureBytes).digest('hex')).toBe(FIXTURE_SHA256) + expect(merkleVectors.map(vector => vector.name)).toEqual( + expect.arrayContaining(['even-four', 'odd-five', 'unsorted-asymmetric-byte-values']) + ) + }) + + it.each(fixture.byteOrder)('reverses display/internal bytes for %s', vector => { + expect(displayToInternal(vector.display).toString('hex')).toBe(vector.internal) + expect(internalToDisplay(Buffer.from(vector.internal, 'hex'))).toBe(vector.display) + }) + + it.each(merkleVectors)('independently hashes the ordered admitted list for %s', vector => { + const independent = independentBasmRoot(vector.txids, sha256d) + expect(independent).toBe(vector.root) + expect(computeBasmRoot(vector.txids)).toBe(independent) + if (vector.txids.length >= 2 && opensslAvailable) { + expect(independentBasmRoot(vector.txids, opensslSha256d)).toBe(independent) + } + const admitted = vector.txids.map((txid, blockIndex) => ({ txid, blockIndex })) + if (vector.admissionListValid) { + expect(basmAdmitted(admitted, admitted.length + 1)).toEqual(admitted) + } else { + expect(() => basmAdmitted(admitted, admitted.length + 1)).toThrow('unique txids') + } + }) + + it.each(tacVectors)('independently chains every TAC step for %s', vector => { + const rootsByName = new Map(merkleVectors.map(root => [root.name, root])) + let previousTac = BASM_ZERO_HASH + let expectedHeight = vector.genesisHeight + + for (const anchor of vector.anchors) { + expect(anchor.blockHeight).toBe(expectedHeight) + const source = rootsByName.get(anchor.rootSource) + expect(source).toBeDefined() + expect(anchor.basmRoot).toBe(source?.root) + expect(anchor.admittedCount).toBe(source?.txids.length) + const independent = independentTac(previousTac, anchor.blockHash, anchor.basmRoot, sha256d) + expect(independent).toBe(anchor.expectedTac) + expect(computeTac(previousTac, anchor.blockHash, anchor.basmRoot)).toBe(independent) + if (opensslAvailable) { + expect(independentTac(previousTac, anchor.blockHash, anchor.basmRoot, opensslSha256d)).toBe( + independent + ) + } + previousTac = independent + expectedHeight += 1 + } + }) +}) diff --git a/packages/overlays/overlay/src/__tests/BASMGoInterop.test.ts b/packages/overlays/overlay/src/__tests/BASMGoInterop.test.ts new file mode 100644 index 000000000..42169f66e --- /dev/null +++ b/packages/overlays/overlay/src/__tests/BASMGoInterop.test.ts @@ -0,0 +1,214 @@ +import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from 'node:child_process' +import { createHash } from 'node:crypto' +import { copyFileSync, existsSync, mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { MerklePath, Transaction } from '@bsv/sdk' +import { BASMRemote } from '../BASMRemote' +import { computeBasmRoot, computeTac, BASM_ZERO_HASH } from '../BASM' + +const TOPIC = 'tm_interop' +const GENESIS_TXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b' +const GENESIS_HEX = + '01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000' +const GENESIS_BLOCK = '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' +const MISSING = 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff' +const GO_SOURCE = join(__dirname, 'fixtures', 'basm-go-read-server.go') + +function displayToInternal(hash: string): Buffer { + return Buffer.from(hash, 'hex').reverse() +} + +function independentTac(previous: string, blockHash: string, root: string): string { + const input = Buffer.concat([ + displayToInternal(previous), + displayToInternal(blockHash), + displayToInternal(root) + ]) + const first = createHash('sha256').update(input).digest() + return Buffer.from(createHash('sha256').update(first).digest()).reverse().toString('hex') +} + +function resolveGoWorktree(): string | undefined { + const candidates = [ + process.env.BASM_GO_OVERLAY_SERVICES, + '/Users/personal/git/go/worktrees/go-overlay-services-basm' + ] + for (const candidate of candidates) { + if ( + candidate !== undefined && + existsSync(join(candidate, 'pkg/server/server_http_basm_interop_test.go')) && + existsSync(join(candidate, 'pkg/core/engine/basm-read-service.go')) + ) { + return candidate + } + } + return undefined +} + +async function waitForUrls( + child: ChildProcessWithoutNullStreams, + timeoutMs: number +): Promise<{ ready: string; unsupported: string }> { + let stdout = '' + let stderr = '' + return await new Promise((resolve, reject) => { + let settled = false + const finish = (error?: Error, urls?: { ready: string; unsupported: string }): void => { + if (settled) return + settled = true + clearTimeout(timer) + child.stdout.off('data', onStdout) + child.stderr.off('data', onStderr) + child.off('error', onError) + child.off('exit', onExit) + if (error !== undefined) reject(error) + else resolve(urls as { ready: string; unsupported: string }) + } + const timer = setTimeout(() => { + finish( + new Error( + `Go BASM host did not become ready: stdout=${stdout.trim()} stderr=${stderr.trim()}` + ) + ) + }, timeoutMs) + const onStdout = (chunk: Buffer | string): void => { + stdout += String(chunk) + const ready = stdout.match(/^READY (http:\/\/127\.0\.0\.1:\d+)/m)?.[1] + const unsupported = stdout.match(/^UNSUPPORTED (http:\/\/127\.0\.0\.1:\d+)/m)?.[1] + if (ready !== undefined && unsupported !== undefined) { + finish(undefined, { ready, unsupported }) + } + } + const onStderr = (chunk: Buffer | string): void => { + stderr += String(chunk) + } + const onError = (error: Error): void => { + finish(error) + } + const onExit = (code: number | null, signal: NodeJS.Signals | null): void => { + finish( + new Error( + `Go BASM host exited code=${String(code)} signal=${String(signal)} stderr=${stderr.trim()}` + ) + ) + } + child.stdout.on('data', onStdout) + child.stderr.on('data', onStderr) + child.on('error', onError) + child.on('exit', onExit) + }) +} + +function startGoHost(goRoot: string): ChildProcessWithoutNullStreams { + const dir = mkdtempSync(join(tmpdir(), 'basm-go-interop-')) + copyFileSync(GO_SOURCE, join(dir, 'main.go')) + writeFileSync( + join(dir, 'go.mod'), + [ + 'module ts-basm-go-interop', + '', + 'go 1.26.0', + '', + 'require github.com/bsv-blockchain/go-overlay-services v0.0.0', + '', + `replace github.com/bsv-blockchain/go-overlay-services => ${goRoot}`, + '' + ].join('\n') + ) + const env = { ...process.env, GOTOOLCHAIN: 'local' } + const tidy = spawnSync('go', ['mod', 'tidy'], { cwd: dir, encoding: 'utf8', env }) + if (tidy.status !== 0) { + throw new Error(`go mod tidy failed: ${tidy.stderr || tidy.stdout}`) + } + const build = spawnSync('go', ['build', '-o', 'basm-host', '.'], { + cwd: dir, + encoding: 'utf8', + env + }) + if (build.status !== 0) { + throw new Error(`go build failed: ${build.stderr || build.stdout}`) + } + const child = spawn(join(dir, 'basm-host'), [], { + cwd: dir, + env, + stdio: ['ignore', 'pipe', 'pipe'] + }) + child.unref() + return child +} + +const goRoot = resolveGoWorktree() +const describeInterop = goRoot === undefined ? describe.skip : describe + +describeInterop('BASMRemote localhost interop against Go read/serving', () => { + let child: ChildProcessWithoutNullStreams + let readyURL = '' + let unsupportedURL = '' + + beforeAll(async () => { + child = startGoHost(goRoot as string) + const urls = await waitForUrls(child, 120000) + readyURL = urls.ready + unsupportedURL = urls.unsupported + }, 130000) + + afterAll(() => { + child?.kill('SIGTERM') + }) + + it('exchanges the five BRC-136 read methods over credential-free HTTP', async () => { + const remote = new BASMRemote(readyURL, TOPIC) + const tip = await remote.requestTopicAnchorTip() + const range = await remote.requestTopicAnchorRange(0, 0) + const admitted = await remote.requestAdmittedList(0, GENESIS_BLOCK) + const proof = await remote.requestCompoundMerklePath(0, [GENESIS_TXID]) + const raw = await remote.requestRawTransactions([GENESIS_TXID, MISSING]) + + expect(tip.topic).toBe(TOPIC) + expect(tip.blockHeight).toBe(0) + expect(tip.blockHash).toBe(GENESIS_BLOCK) + expect(tip.basmRoot).toBe(GENESIS_TXID) + expect(tip.admittedCount).toBe(1) + expect(computeBasmRoot([{ txid: GENESIS_TXID, blockIndex: 0 }])).toBe(GENESIS_TXID) + expect(independentTac(BASM_ZERO_HASH, GENESIS_BLOCK, GENESIS_TXID)).toBe(tip.tac) + expect(computeTac(BASM_ZERO_HASH, GENESIS_BLOCK, GENESIS_TXID)).toBe(tip.tac) + + expect(range.anchors).toHaveLength(1) + expect(range.anchors[0]?.tac).toBe(tip.tac) + expect(range.anchors[0]?.blockHash).toBe(GENESIS_BLOCK) + expect(admitted.admitted).toEqual([{ txid: GENESIS_TXID, blockIndex: 0 }]) + + const parsedProof = MerklePath.fromHex(proof.merklePath) + expect(parsedProof.blockHeight).toBe(0) + expect(parsedProof.path[0]?.[0]?.hash).toBe(GENESIS_TXID) + expect(parsedProof.path[0]?.[0]?.offset).toBe(0) + expect(parsedProof.computeRoot(GENESIS_TXID)).toBe(GENESIS_TXID) + expect(parsedProof.toHex()).toBe(proof.merklePath.toLowerCase()) + + const parsedRaw = Transaction.fromHex(raw.transactions[0].rawTx) + expect(raw.transactions[0].txid).toBe(GENESIS_TXID) + expect(raw.transactions[0].rawTx).toBe(GENESIS_HEX) + expect(parsedRaw.id('hex')).toBe(GENESIS_TXID) + expect(raw.missing).toEqual([MISSING]) + + const cors = await fetch(new URL('/requestTopicAnchorTip', readyURL), { + method: 'POST', + headers: { + Origin: 'https://unknown-wallet.example', + Accept: 'application/json', + 'Content-Type': 'application/json', + 'x-bsv-topic': TOPIC + }, + body: '{}' + }) + expect(cors.ok).toBe(true) + expect(cors.headers.get('access-control-allow-origin')).toBe('*') + + await expect( + new BASMRemote(unsupportedURL, TOPIC).requestTopicAnchorTip() + ).rejects.toMatchObject({ + code: 'BASM_UNSUPPORTED' + }) + }, 30000) +}) diff --git a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts new file mode 100644 index 000000000..9545472e2 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts @@ -0,0 +1,357 @@ +import { createHash } from 'node:crypto' +import { LockingScript, MerklePath, Transaction } from '@bsv/sdk' +import { Engine } from '../Engine' +import { BASM_ZERO_HASH, computeBasmRoot, computeTac } from '../BASM' +import type { TopicBlockAnchor } from '../BASM' +import type { Storage } from '../storage/Storage' + +const topic = 'tm_basm_test' +const blockHeight = 100 +const blockHash = 'ab'.repeat(32) +const hashPair = (left: string, right: string): string => { + const bytes = Buffer.concat([ + Buffer.from(left, 'hex').reverse(), + Buffer.from(right, 'hex').reverse() + ]) + const first = createHash('sha256').update(bytes).digest() + return createHash('sha256').update(first).digest().reverse().toString('hex') +} + +function fixture() { + const transactions = [1, 2, 3, 4].map( + satoshis => + new Transaction(1, [], [{ satoshis, lockingScript: LockingScript.fromASM('OP_TRUE') }], 0) + ) + const ids = transactions.map(tx => tx.id('hex')) + const root = hashPair(hashPair(ids[0], ids[1]), hashPair(ids[2], ids[3])) + const path = new MerklePath(blockHeight, [ + ids.map((hash, offset) => ({ offset, hash, txid: true })), + [] + ]) + const admitted = [ + { txid: ids[1], blockIndex: 1 }, + { txid: ids[3], blockIndex: 3 } + ] + const anchor: TopicBlockAnchor = { + topic, + blockHeight, + blockHash, + basmRoot: computeBasmRoot(admitted), + admittedCount: 2, + tac: '' + } + anchor.tac = computeTac(BASM_ZERO_HASH, blockHash, anchor.basmRoot) + const responses: Record = { + '/requestTopicAnchorTip': anchor, + '/requestTopicAnchorRange': { topic, anchors: [anchor] }, + '/requestAdmittedList': { topic, blockHeight, blockHash, admitted }, + '/requestCompoundMerklePath': { + topic, + blockHeight, + txids: admitted.map(item => item.txid), + merklePath: path.toHex() + }, + '/requestRawTransactions': { + // Deliberately reversed: admission must still follow original block order. + transactions: [3, 1].map(index => ({ txid: ids[index], rawTx: transactions[index].toHex() })), + missing: [] + } + } + const storage = { + findTopicAnchorTip: jest.fn(async () => undefined), + findTopicBlockAnchor: jest.fn(async () => undefined), + findAdmittedTransactionsForBlock: jest.fn(async () => []) + } + const tracker = { + currentHeight: jest.fn(async () => 250), + isValidRootForHeight: jest.fn( + async (candidate: string, height: number) => candidate === root && height === blockHeight + ) + } + const engine = new Engine({}, {}, storage as unknown as Storage, tracker) + engine.syncConfiguration = { [topic]: ['https://peer.example'] } + engine.topicAnchorHeaderResolver = jest.fn(async height => ({ + blockHeight: height, + blockHash, + merkleRoot: root + })) + engine.logger = { ...console, error: jest.fn() } + const submit = jest.spyOn(engine, 'submit').mockResolvedValue({}) + const requests: Array<{ path: string; body: Record }> = [] + const fetchMock = jest.spyOn(globalThis, 'fetch').mockImplementation(async (url, init) => { + const endpoint = new URL(String(url)).pathname + requests.push({ + path: endpoint, + body: JSON.parse(String(init?.body)) as Record + }) + return new Response(JSON.stringify(responses[endpoint]), { status: 200 }) + }) + return { + engine, + responses, + anchor, + admitted, + transactions, + ids, + path, + submit, + storage, + tracker, + requests, + fetchMock + } +} + +afterEach(() => jest.restoreAllMocks()) + +describe('BASM reconciliation evidence binding', () => { + it('verifies a multi-level subset with original index gaps and submits historically in block order', async () => { + const f = fixture() + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('advanced') + expect(report.fetchedTxCount).toBe(2) + expect(report.positionValidation).toBe('encoded-offset-only') + expect(f.submit).toHaveBeenCalledTimes(2) + expect( + f.submit.mock.calls.map(([tagged]) => Transaction.fromBEEF(tagged.beef).id('hex')) + ).toEqual([f.ids[1], f.ids[3]]) + for (const [, callback, mode] of f.submit.mock.calls) { + expect(callback).toBeUndefined() + expect(mode).toBe('historical-tx') + } + expect(f.tracker.isValidRootForHeight).toHaveBeenCalledTimes(2) + }) + + it('reports stronger position validation only when a trusted count is bound to the canonical header', async () => { + const f = fixture() + f.engine.topicAnchorHeaderResolver = async height => ({ + blockHeight: height, + blockHash, + blockTransactionCount: 4 + }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('advanced') + expect(report.positionValidation).toBe('canonical-count') + expect(f.submit).toHaveBeenCalledTimes(2) + }) + + it.each([0, -1, 1, 1.5, 3, Number.MAX_SAFE_INTEGER + 1, Number.NaN])( + 'rejects inconsistent canonical full-block count %s', + async blockTransactionCount => { + const f = fixture() + f.engine.topicAnchorHeaderResolver = async height => ({ + blockHeight: height, + blockHash, + blockTransactionCount + }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(f.submit).not.toHaveBeenCalled() + } + ) + + it('rejects an admitted phantom duplicate at the right edge of an odd-width block', async () => { + const f = fixture() + const duplicated = [f.ids[0], f.ids[1], f.ids[2], f.ids[2]] + const root = hashPair( + hashPair(duplicated[0], duplicated[1]), + hashPair(duplicated[2], duplicated[3]) + ) + const proof = new MerklePath(blockHeight, [ + duplicated.map((hash, offset) => ({ hash, offset, txid: true })), + [] + ]) + const admitted = [{ txid: f.ids[2], blockIndex: 3 }] + const anchor = { + ...f.anchor, + basmRoot: f.ids[2], + admittedCount: 1, + tac: computeTac(BASM_ZERO_HASH, blockHash, f.ids[2]) + } + f.responses['/requestTopicAnchorTip'] = anchor + f.responses['/requestTopicAnchorRange'] = { topic, anchors: [anchor] } + f.responses['/requestAdmittedList'] = { topic, blockHeight, blockHash, admitted } + f.responses['/requestCompoundMerklePath'] = { + topic, + blockHeight, + txids: [f.ids[2]], + merklePath: proof.toHex() + } + f.engine.topicAnchorHeaderResolver = async height => ({ + blockHeight: height, + blockHash, + merkleRoot: root, + blockTransactionCount: 3 + }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.message).toContain('canonical block transaction count') + expect(f.submit).not.toHaveBeenCalled() + }) + + it.each([ + 'block hash', + 'header height', + 'header unavailable', + 'header root', + 'proof height', + 'proof index', + 'raw identity', + 'raw trailing bytes', + 'range TAC', + 'range gap' + ])('rejects invalid %s before admission', async failure => { + const f = fixture() + switch (failure) { + case 'block hash': + f.engine.topicAnchorHeaderResolver = async height => ({ + blockHeight: height, + blockHash: 'cd'.repeat(32) + }) + break + case 'header height': + f.engine.topicAnchorHeaderResolver = async height => ({ + blockHeight: height + 1, + blockHash + }) + break + case 'header unavailable': + f.engine.topicAnchorHeaderResolver = async () => undefined + break + case 'header root': + f.engine.topicAnchorHeaderResolver = async height => ({ + blockHeight: height, + blockHash, + merkleRoot: BASM_ZERO_HASH + }) + break + case 'proof height': + f.path.blockHeight += 1 + f.responses['/requestCompoundMerklePath'] = { + topic, + blockHeight, + txids: f.admitted.map(item => item.txid), + merklePath: f.path.toHex() + } + break + case 'proof index': + f.admitted[0].blockIndex = 0 + break + case 'raw identity': + case 'raw trailing bytes': + f.responses['/requestRawTransactions'] = { + transactions: f.admitted.map((item, i) => ({ + txid: item.txid, + rawTx: + f.transactions[failure === 'raw identity' ? 0 : i * 2 + 1].toHex() + + (failure === 'raw trailing bytes' ? '00' : '') + })), + missing: [] + } + break + case 'range TAC': + f.anchor.tac = 'cd'.repeat(32) + break + case 'range gap': + f.responses['/requestTopicAnchorRange'] = { topic, anchors: [] } + break + } + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(f.submit).not.toHaveBeenCalled() + }) + + it('requires canonical header capability even for equal-height matching tips', async () => { + const f = fixture() + jest.spyOn(f.engine, 'provideTopicAnchorTip').mockResolvedValue(f.anchor) + f.engine.topicAnchorHeaderResolver = undefined + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.message).toContain('canonical header resolver') + expect(f.submit).not.toHaveBeenCalled() + }) + + it('detects a peer anchor changing while raw transactions are fetched', async () => { + const f = fixture() + const originalFetch = f.fetchMock.getMockImplementation()! + f.fetchMock.mockImplementation(async (url, init) => { + const response = await originalFetch(url, init) + if (new URL(String(url)).pathname === '/requestRawTransactions') { + f.responses['/requestTopicAnchorRange'] = { + topic, + anchors: [{ ...f.anchor, tac: 'ef'.repeat(32) }] + } + } + return response + }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.message).toContain('changed before admission') + expect(f.submit).not.toHaveBeenCalled() + }) + + it('starts the next forward page immediately after the local tip when remote is over a page ahead', async () => { + const f = fixture() + jest + .spyOn(f.engine, 'provideTopicAnchorTip') + .mockResolvedValue({ topic, blockHeight: 1, tac: BASM_ZERO_HASH }) + f.anchor.blockHeight = 3000 + await f.engine.startBASMSync() + const ranges = f.requests.filter(request => request.path === '/requestTopicAnchorRange') + expect(ranges.map(request => request.body)).toEqual([ + { fromHeight: 3000, toHeight: 3000 }, + { fromHeight: 2, toHeight: 1001 } + ]) + expect(f.submit).not.toHaveBeenCalled() + }) + + it('refuses an old-topic bootstrap tail without a trusted TAC prefix (B02 continuation required)', async () => { + const f = fixture() + const anchors: TopicBlockAnchor[] = [] + let tac = BASM_ZERO_HASH + for (let height = 100; height <= 1100; height++) { + const basmRoot = height === 100 ? f.anchor.basmRoot : BASM_ZERO_HASH + tac = computeTac(tac, blockHash, basmRoot) + anchors.push({ + topic, + blockHeight: height, + blockHash, + basmRoot, + admittedCount: height === 100 ? 2 : 0, + tac + }) + } + f.fetchMock.mockImplementation(async (url, init) => { + const endpoint = new URL(String(url)).pathname + if (endpoint === '/requestTopicAnchorTip') return Response.json(anchors.at(-1)) + const body = JSON.parse(String(init?.body)) as { fromHeight: number; toHeight: number } + return Response.json({ + topic, + anchors: anchors.filter( + anchor => anchor.blockHeight >= body.fromHeight && anchor.blockHeight <= body.toHeight + ) + }) + }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.message).toContain('TAC is inconsistent with its prefix') + expect(f.submit).not.toHaveBeenCalled() + }) + + it('reports a finite proof request limit for a block above 1000 admissions (B02 chunking required)', async () => { + const f = fixture() + const admitted = Array.from({ length: 1001 }, (_, blockIndex) => ({ + txid: blockIndex.toString(16).padStart(64, '0'), + blockIndex + })) + f.anchor.admittedCount = admitted.length + f.anchor.basmRoot = computeBasmRoot(admitted) + f.anchor.tac = computeTac(BASM_ZERO_HASH, blockHash, f.anchor.basmRoot) + f.responses['/requestAdmittedList'] = { topic, blockHeight, blockHash, admitted } + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.errorCode).toBe('BASM_RESOURCE_LIMIT') + expect(f.requests.some(request => request.path === '/requestCompoundMerklePath')).toBe(false) + expect(f.submit).not.toHaveBeenCalled() + }) +}) diff --git a/packages/overlays/overlay/src/__tests/BASMRemote.test.ts b/packages/overlays/overlay/src/__tests/BASMRemote.test.ts new file mode 100644 index 000000000..4b546f24c --- /dev/null +++ b/packages/overlays/overlay/src/__tests/BASMRemote.test.ts @@ -0,0 +1,380 @@ +import { BASMProtocolError, BASMRemote } from '../BASMRemote' + +const ENDPOINT = 'https://peer.example/overlay' +const TOPIC = 'tm_example' +const ZERO = '0000000000000000000000000000000000000000000000000000000000000000' +const TXID_1 = '0101010101010101010101010101010101010101010101010101010101010101' +const TXID_2 = '0202020202020202020202020202020202020202020202020202020202020202' +const TXID_3 = '0303030303030303030303030303030303030303030303030303030303030303' +const BLOCK_HASH = '0404040404040404040404040404040404040404040404040404040404040404' +const BASM_ROOT = '0505050505050505050505050505050505050505050505050505050505050505' +const TAC = '0606060606060606060606060606060606060606060606060606060606060606' + +function response(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'content-type': 'application/json' } + }) +} + +function remoteFor(value: unknown, limits = {}): BASMRemote { + return new BASMRemote(ENDPOINT, TOPIC, async () => response(value), limits) +} + +function anchor(blockHeight: number): Record { + return { + topic: TOPIC, + blockHeight, + blockHash: BLOCK_HASH, + basmRoot: BASM_ROOT, + admittedCount: 0, + tac: TAC + } +} + +describe('BASMRemote', () => { + it('preserves the BASM request URL and wire shape while accepting legacy minimal responses', async () => { + const calls: Array<[RequestInfo | URL, RequestInit | undefined]> = [] + const injectedFetch: typeof fetch = async (input, init) => { + calls.push([input, init]) + if (String(input).endsWith('/requestTopicAnchorTip')) { + return response({ topic: TOPIC, blockHeight: -1, tac: ZERO }) + } + return response({ topic: TOPIC, blockHeight: 42, admitted: [] }) + } + const remote = new BASMRemote(ENDPOINT, TOPIC, injectedFetch) + + await expect(remote.requestTopicAnchorTip()).resolves.toEqual({ + topic: TOPIC, + blockHeight: -1, + tac: ZERO + }) + await expect(remote.requestAdmittedList(42)).resolves.toEqual({ + topic: TOPIC, + blockHeight: 42, + admitted: [] + }) + expect(calls).toEqual([ + [ + 'https://peer.example/requestTopicAnchorTip', + { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + 'x-bsv-topic': TOPIC + }, + body: '{}', + signal: expect.any(AbortSignal) + } + ], + [ + 'https://peer.example/requestAdmittedList', + { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + 'x-bsv-topic': TOPIC + }, + body: JSON.stringify({ blockHeight: 42 }), + signal: expect.any(AbortSignal) + } + ] + ]) + }) + + it.each([ + ['wrong topic', { topic: 'tm_other', blockHeight: 7, tac: TAC }], + ['wrong height', { topic: TOPIC, blockHeight: 8, admitted: [] }], + [ + 'conflicting block hash', + { topic: TOPIC, blockHeight: 7, blockHash: BLOCK_HASH, admitted: [] } + ] + ])('rejects an admitted response with a %s', async (_label, value) => { + const remote = remoteFor(value) + await expect(remote.requestAdmittedList(7, TXID_1)).rejects.toMatchObject({ + name: 'BASMProtocolError', + code: 'BASM_INVALID_RESPONSE' + }) + }) + + it('caps the admitted list before accepting unbounded peer data', async () => { + const remote = remoteFor( + { + topic: TOPIC, + blockHeight: 7, + admitted: [ + { txid: TXID_1, blockIndex: 0 }, + { txid: TXID_2, blockIndex: 1 } + ] + }, + { maxAdmittedTxids: 1 } + ) + + await expect(remote.requestAdmittedList(7)).rejects.toMatchObject({ + code: 'BASM_RESOURCE_LIMIT' + }) + }) + + it.each([ + ['unordered anchors', [anchor(5), anchor(4)]], + ['a gap between anchors', [anchor(4), anchor(6)]] + ])('rejects an anchor range with %s', async (_label, anchors) => { + const remote = remoteFor({ topic: TOPIC, anchors }) + await expect(remote.requestTopicAnchorRange(4, 6)).rejects.toMatchObject({ + code: 'BASM_INVALID_RESPONSE' + }) + }) + + it('rejects an anchor response with more entries than the requested range', async () => { + const remote = remoteFor({ topic: TOPIC, anchors: [anchor(4), anchor(5), anchor(6)] }) + await expect(remote.requestTopicAnchorRange(4, 5)).rejects.toMatchObject({ + code: 'BASM_RESOURCE_LIMIT' + }) + }) + + it.each([ + [ + 'a duplicate txid', + [ + { txid: TXID_1, blockIndex: 0 }, + { txid: TXID_1, blockIndex: 1 } + ] + ], + [ + 'a duplicate block index', + [ + { txid: TXID_1, blockIndex: 0 }, + { txid: TXID_2, blockIndex: 0 } + ] + ], + [ + 'nonmonotonic block indices', + [ + { txid: TXID_1, blockIndex: 2 }, + { txid: TXID_2, blockIndex: 1 } + ] + ] + ])('rejects an admitted list with %s', async (_label, admitted) => { + const remote = remoteFor({ topic: TOPIC, blockHeight: 7, admitted }) + await expect(remote.requestAdmittedList(7)).rejects.toMatchObject({ + code: 'BASM_INVALID_RESPONSE' + }) + }) + + it('accepts a proof response whose txids are reordered but exactly match the request', async () => { + const remote = remoteFor({ + topic: TOPIC, + blockHeight: 7, + txids: [TXID_2, TXID_1], + merklePath: 'aabb' + }) + await expect(remote.requestCompoundMerklePath(7, [TXID_1, TXID_2])).resolves.toEqual({ + topic: TOPIC, + blockHeight: 7, + txids: [TXID_2, TXID_1], + merklePath: 'aabb' + }) + }) + + it('rejects a proof response that does not contain exactly the requested txid set', async () => { + const remote = remoteFor({ + topic: TOPIC, + blockHeight: 7, + txids: [TXID_1, TXID_3], + merklePath: 'aabb' + }) + await expect(remote.requestCompoundMerklePath(7, [TXID_1, TXID_2])).rejects.toMatchObject({ + code: 'BASM_INVALID_RESPONSE' + }) + }) + + it.each([ + [ + 'an extra transaction', + { transactions: [{ txid: TXID_3, rawTx: 'aabb' }], missing: [TXID_1, TXID_2] } + ], + ['an omitted transaction', { transactions: [{ txid: TXID_1, rawTx: 'aabb' }], missing: [] }], + [ + 'a duplicate transaction', + { + transactions: [ + { txid: TXID_1, rawTx: 'aabb' }, + { txid: TXID_1, rawTx: 'ccdd' } + ], + missing: [] + } + ], + [ + 'a transaction also reported missing', + { transactions: [{ txid: TXID_1, rawTx: 'aabb' }], missing: [TXID_1, TXID_2] } + ] + ])('rejects a raw transaction response containing %s', async (_label, value) => { + const remote = remoteFor(value) + await expect(remote.requestRawTransactions([TXID_1, TXID_2])).rejects.toMatchObject({ + code: 'BASM_INVALID_RESPONSE' + }) + }) + + it('rejects a chunked response once its streamed bytes cross the configured cap', async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(Buffer.from('{"topic"')) + controller.enqueue(Buffer.from(':"tm_example"}')) + controller.close() + } + }) + const remote = new BASMRemote(ENDPOINT, TOPIC, async () => new Response(stream), { + maxResponseBytes: 10 + }) + + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ + code: 'BASM_RESOURCE_LIMIT' + }) + }) + + it('rejects an advertised response length above the configured cap before reading it', async () => { + const remote = new BASMRemote( + ENDPOINT, + TOPIC, + async () => new Response('{"topic":"tm_example"}', { headers: { 'content-length': '11' } }), + { maxResponseBytes: 10 } + ) + + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ + code: 'BASM_RESOURCE_LIMIT' + }) + }) + + it('reports malformed successful JSON as a protocol error', async () => { + const remote = new BASMRemote(ENDPOINT, TOPIC, async () => new Response('{'), {}) + await expect(remote.requestTopicAnchorTip()).rejects.toBeInstanceOf(BASMProtocolError) + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ + code: 'BASM_INVALID_RESPONSE' + }) + }) + + it('times out a fetch that observes the abort signal', async () => { + let abortObserved = false + const remote = new BASMRemote( + ENDPOINT, + TOPIC, + async (_input, init) => + await new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + abortObserved = true + reject(init.signal?.reason) + }) + }), + { timeoutMs: 20 } + ) + + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ code: 'BASM_TIMEOUT' }) + expect(abortObserved).toBe(true) + }) + + it('times out a fetch that ignores the abort signal', async () => { + let signal: AbortSignal | undefined + const remote = new BASMRemote( + ENDPOINT, + TOPIC, + async (_input, init) => + await new Promise(() => { + signal = init?.signal ?? undefined + }), + { timeoutMs: 20 } + ) + + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ code: 'BASM_TIMEOUT' }) + expect(signal?.aborted).toBe(true) + }) + + it('times out while a real response body stalls', async () => { + const stalled = new ReadableStream({ + pull: async () => await new Promise(() => {}) + }) + const remote = new BASMRemote(ENDPOINT, TOPIC, async () => new Response(stalled), { + timeoutMs: 20 + }) + + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ code: 'BASM_TIMEOUT' }) + }) + + it.each([ + ['a 501 response', response({}, 501), 'BASM_UNSUPPORTED'], + [ + 'an explicit unsupported response', + response({ code: 'BASM_UNSUPPORTED' }, 400), + 'BASM_UNSUPPORTED' + ], + ['a generic 503 response', response({}, 503), 'BASM_HTTP_ERROR'] + ])('classifies %s', async (_label, peerResponse, code) => { + const remote = new BASMRemote(ENDPOINT, TOPIC, async () => peerResponse) + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ code }) + }) + + it('accepts a contiguous range and a complete raw-transaction partition', async () => { + const remote = new BASMRemote(ENDPOINT, TOPIC, async input => { + if (String(input).endsWith('/requestTopicAnchorRange')) { + return response({ topic: TOPIC, anchors: [anchor(4), anchor(5)] }) + } + return response({ + transactions: [{ txid: TXID_2, rawTx: 'ccdd' }], + missing: [TXID_1] + }) + }) + + await expect(remote.requestTopicAnchorRange(4, 5)).resolves.toEqual({ + topic: TOPIC, + anchors: [anchor(4), anchor(5)] + }) + await expect(remote.requestRawTransactions([TXID_1, TXID_2])).resolves.toEqual({ + transactions: [{ txid: TXID_2, rawTx: 'ccdd' }], + missing: [TXID_1] + }) + }) + + it.each([ + [ + 'an uppercase TAC', + async (remote: BASMRemote) => await remote.requestTopicAnchorTip(), + { topic: TOPIC, blockHeight: -1, tac: 'ab'.repeat(32).toUpperCase() } + ], + [ + 'a numeric-string height', + async (remote: BASMRemote) => await remote.requestTopicAnchorTip(), + { topic: TOPIC, blockHeight: '7', tac: TAC } + ], + [ + 'a mixed-case block hash', + async (remote: BASMRemote) => await remote.requestTopicAnchorRange(4, 4), + { topic: TOPIC, anchors: [{ ...anchor(4), blockHash: 'cd'.repeat(32).toUpperCase() }] } + ] + ])('rejects %s rather than coercing untrusted peer JSON', async (_label, invoke, value) => { + await expect(invoke(remoteFor(value))).rejects.toMatchObject({ + code: 'BASM_INVALID_RESPONSE' + }) + }) + + it('rejects a duplicated height as an unordered or gapped range', async () => { + const remote = remoteFor({ topic: TOPIC, anchors: [anchor(4), anchor(4)] }) + await expect(remote.requestTopicAnchorRange(4, 5)).rejects.toMatchObject({ + code: 'BASM_INVALID_RESPONSE' + }) + }) + + it('ignores unknown additive JSON fields on a legacy tip', async () => { + const remote = remoteFor({ + topic: TOPIC, + blockHeight: -1, + tac: ZERO, + extra: 'ignored' + }) + await expect(remote.requestTopicAnchorTip()).resolves.toEqual({ + topic: TOPIC, + blockHeight: -1, + tac: ZERO + }) + }) +}) diff --git a/packages/overlays/overlay/src/__tests/fixtures/README.md b/packages/overlays/overlay/src/__tests/fixtures/README.md new file mode 100644 index 000000000..4ad06bdb3 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/fixtures/README.md @@ -0,0 +1,26 @@ +# Independent BRC-136 fixture + +brc136-independent.json is copied byte-for-byte from the Go BASM +foundation fixture: + +/Users/personal/git/go/worktrees/go-overlay-services-basm/pkg/core/basm/testdata/vectors.json + +Source commit: d99216814a4b9dca5f9f4d04a602ef2d48bdc4a7 +Go worktree revision used for inventory: faaf69d372fd5e9974eaadbae9b8e26d761f0c86 +BRC-136 revision: 2733cd2950a739b3c977b95d652ff63e3773c40b +Fixture SHA-256: 51983432bb561e3031fb7c983947dc8bfdcfd0ef828ef9f24093666ff4665c8a + +The vectors cover ordered admitted subsets, multi-level Merkle roots, +Bitcoin odd-node duplication, original display/internal byte order, and +contiguous TAC chains through empty heights. The +repeated-last-four-illustrative-invalid vector intentionally shares a root +with the three-leaf illustrative vector. Primitive root calculation accepts +that input, while admitted-list validation must reject duplicate transaction +IDs. This fixture does not define multiproof wire bytes. + +TypeScript tests recompute every root and TAC with Node crypto SHA-256d and, +when present, OpenSSL `dgst -sha256`; they do not treat the Go expected +values as authoritative without that independent check. +`basm-go-read-server.go` is a local B01 interop host compiled into a temporary +module. It replaces onto the Go overlay-services worktree and must not modify +that tree's uncommitted S04 files. diff --git a/packages/overlays/overlay/src/__tests/fixtures/basm-go-read-server.go b/packages/overlays/overlay/src/__tests/fixtures/basm-go-read-server.go new file mode 100644 index 000000000..63f1e6f7e --- /dev/null +++ b/packages/overlays/overlay/src/__tests/fixtures/basm-go-read-server.go @@ -0,0 +1,211 @@ +// Temporary B01 interop host. Compiled from the TypeScript test into a temp +// module that replaces onto the local go-overlay-services worktree. It does not +// live in that worktree and must not be used to edit S04 files. +package main + +import ( + "context" + "encoding/hex" + "fmt" + "net" + "os" + "os/signal" + "syscall" + + "github.com/bsv-blockchain/go-sdk/chainhash" + "github.com/bsv-blockchain/go-sdk/overlay" + "github.com/bsv-blockchain/go-sdk/overlay/lookup" + "github.com/bsv-blockchain/go-sdk/transaction" + "github.com/gofiber/fiber/v2" + + "github.com/bsv-blockchain/go-overlay-services/pkg/core/basm" + "github.com/bsv-blockchain/go-overlay-services/pkg/core/engine" + "github.com/bsv-blockchain/go-overlay-services/pkg/core/gasp" + "github.com/bsv-blockchain/go-overlay-services/pkg/server" +) + +const ( + interopTopic = "tm_interop" + interopGenesisID = "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b" + interopGenesisHex = "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000" + interopGenesisBlock = "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + ready, err := newReadyService() + if err != nil { + return err + } + readyURL, err := listen(ready) + if err != nil { + return err + } + unsupportedURL, err := listen(nil) + if err != nil { + return err + } + fmt.Printf("READY %s\n", readyURL) + fmt.Printf("UNSUPPORTED %s\n", unsupportedURL) + _ = os.Stdout.Sync() + + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + <-signals + return nil +} + +func listen(provider engine.BASMProvider) (string, error) { + app := server.RegisterRoutesWithErrorHandler(fiber.New(fiber.Config{DisableStartupMessage: true}), &server.RegisterRoutesConfig{ + AdminBearerToken: "interop-admin-token", + Engine: stubEngine{}, + BASMProvider: provider, + BASMLimits: basm.DefaultReadLimits(), + }) + listener, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0") + if err != nil { + return "", err + } + go func() { + if serveErr := app.Listener(listener); serveErr != nil { + fmt.Fprintln(os.Stderr, serveErr) + } + }() + return "http://" + listener.Addr().String(), nil +} + +func newReadyService() (*engine.BASMReadService, error) { + txid, err := basm.ParseHash(interopGenesisID) + if err != nil { + return nil, err + } + blockHash, err := basm.ParseHash(interopGenesisBlock) + if err != nil { + return nil, err + } + txidFlag := true + transactionHash := chainhash.Hash(txid) + proof := transaction.NewMerklePath(0, [][]*transaction.PathElement{{{ + Offset: 0, + Hash: &transactionHash, + Txid: &txidFlag, + }}}).Bytes() + anchor := basm.Anchor{TopicBlockAnchor: basm.TopicBlockAnchor{ + Topic: interopTopic, + BlockHeight: 0, + BlockHash: blockHash, + BASMRoot: txid, + AdmittedCount: 1, + }} + anchor.TAC = basm.HashTACStep(basm.Hash{}, anchor.BlockHash, anchor.BASMRoot) + storage := interopStorage{view: interopView{anchor: anchor, txid: txid, proof: proof}} + headers := interopHeaders{header: engine.BASMCanonicalHeader{ + Height: 0, + BlockHash: blockHash, + MerkleRoot: txid, + TransactionCount: 1, + }} + return engine.NewBASMReadService(storage, headers, basm.DefaultReadLimits()) +} + +type interopStorage struct{ view interopView } + +func (s interopStorage) OpenBASMRead(_ context.Context, topic string, _ basm.ReadLimits) (engine.BASMReadView, error) { + if topic != "" && topic != interopTopic { + return nil, engine.ErrBASMNotFound + } + return s.view, nil +} + +type interopView struct { + anchor basm.Anchor + txid basm.Hash + proof []byte +} + +func (v interopView) Tip(context.Context) (*basm.Anchor, error) { + anchor := v.anchor + return &anchor, nil +} + +func (v interopView) Anchors(_ context.Context, from, to, _ uint32) ([]basm.Anchor, error) { + if from != 0 || to != 0 { + return nil, engine.ErrBASMNotReady + } + return []basm.Anchor{v.anchor}, nil +} + +func (v interopView) Admitted(_ context.Context, height, _ uint32) ([]basm.AdmittedTxRef, error) { + if height != 0 { + return nil, engine.ErrBASMNotReady + } + return []basm.AdmittedTxRef{{TxID: v.txid, BlockIndex: 0}}, nil +} + +func (v interopView) MerklePath(_ context.Context, txid basm.Hash, _ uint32) ([]byte, error) { + if txid != v.txid { + return nil, engine.ErrBASMNotFound + } + return append([]byte(nil), v.proof...), nil +} + +func (v interopView) RawTx(_ context.Context, txid basm.Hash, _ uint32) ([]byte, error) { + if txid != v.txid { + return nil, engine.ErrBASMNotFound + } + return hex.DecodeString(interopGenesisHex) +} + +func (interopView) CheckCurrent(context.Context) error { return nil } +func (interopView) Close() error { return nil } + +type interopHeaders struct{ header engine.BASMCanonicalHeader } + +func (h interopHeaders) CanonicalBASMHeader(_ context.Context, height uint32) (engine.BASMCanonicalHeader, error) { + if height != h.header.Height { + return engine.BASMCanonicalHeader{}, engine.ErrBASMNotReady + } + return h.header, nil +} + +type stubEngine struct{} + +func (stubEngine) Submit(_ context.Context, _ overlay.TaggedBEEF, _ engine.SumbitMode, onSteakReady engine.OnSteakReady) (overlay.Steak, error) { + if onSteakReady != nil { + onSteakReady(&overlay.Steak{}) + } + return overlay.Steak{}, nil +} +func (stubEngine) Lookup(_ context.Context, _ *lookup.LookupQuestion) (*lookup.LookupAnswer, error) { + return &lookup.LookupAnswer{}, nil +} +func (stubEngine) GetUTXOHistory(_ context.Context, _ *engine.Output, _ func(beef *transaction.Beef, outputIndex, currentDepth uint32) bool, _ uint32) (*engine.Output, error) { + return &engine.Output{}, nil +} +func (stubEngine) SyncAdvertisements(context.Context) error { return nil } +func (stubEngine) StartGASPSync(context.Context) error { return nil } +func (stubEngine) ProvideForeignSyncResponse(_ context.Context, _ *gasp.InitialRequest, _ string) (*gasp.InitialResponse, error) { + return &gasp.InitialResponse{}, nil +} +func (stubEngine) ProvideForeignGASPNode(_ context.Context, _, _ *transaction.Outpoint, _ string) (*gasp.Node, error) { + return &gasp.Node{}, nil +} +func (stubEngine) ListTopicManagers() map[string]*overlay.MetaData { + return map[string]*overlay.MetaData{} +} +func (stubEngine) ListLookupServiceProviders() map[string]*overlay.MetaData { + return map[string]*overlay.MetaData{} +} +func (stubEngine) GetDocumentationForLookupServiceProvider(string) (string, error) { + return "", nil +} +func (stubEngine) GetDocumentationForTopicManager(string) (string, error) { return "", nil } +func (stubEngine) HandleNewMerkleProof(context.Context, *chainhash.Hash, *transaction.MerklePath) error { + return nil +} diff --git a/packages/overlays/overlay/src/__tests/fixtures/brc136-independent.json b/packages/overlays/overlay/src/__tests/fixtures/brc136-independent.json new file mode 100644 index 000000000..1498e2a10 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/fixtures/brc136-independent.json @@ -0,0 +1,185 @@ +{ + "specRevision": { + "brc136": "2733cd2950a739b3c977b95d652ff63e3773c40b", + "repoHead": "39a643ff148a8dcd23ec08986a8ddeb7d5713743" + }, + "hashEncoding": "lowercase display-order hex", + "byteOrder": [ + { + "name": "asymmetric-display-to-internal", + "display": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "internal": "1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100" + } + ], + "merkle": [ + { + "name": "empty", + "txids": [], + "root": "0000000000000000000000000000000000000000000000000000000000000000", + "admissionListValid": true + }, + { + "name": "singleton-genesis-bitcoin-txid", + "txids": [ + "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b" + ], + "root": "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + "admissionListValid": true + }, + { + "name": "singleton-asymmetric-byte-order", + "txids": [ + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + ], + "root": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "admissionListValid": true + }, + { + "name": "even-two-asymmetric", + "txids": [ + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "ffeeddccbbaa99887766554433221100deadbeefcafebabe0123456789abcdef" + ], + "root": "668066643818b3695ed31c7cfd09dcc8aaeba2f6eb46e6360b041fb3254ccffc", + "admissionListValid": true + }, + { + "name": "even-four", + "txids": [ + "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "ffeeddccbbaa99887766554433221100deadbeefcafebabe0123456789abcdef", + "112233445566778899aabbccddeeff00123456789abcdef00fedcba987654321" + ], + "root": "4c59e81df05211158cb925226168b4536aef5b6c7a7575197e09a77234a63114", + "admissionListValid": true + }, + { + "name": "odd-three", + "txids": [ + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "ffeeddccbbaa99887766554433221100deadbeefcafebabe0123456789abcdef", + "112233445566778899aabbccddeeff00123456789abcdef00fedcba987654321" + ], + "root": "82853ce87598bf885ef5fb2e288d92581a1addc09bc651ea5e3bc2834867f7a0", + "admissionListValid": true + }, + { + "name": "tac-three-asymmetric", + "txids": [ + "ffeeddccbbaa99887766554433221100deadbeefcafebabe0123456789abcdef", + "112233445566778899aabbccddeeff00123456789abcdef00fedcba987654321", + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ], + "root": "bd162ecb4feb15a6489b8e7a6112dadfb1a699e691077b53dabf4cf79462173c", + "admissionListValid": true + }, + { + "name": "odd-five", + "txids": [ + "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "ffeeddccbbaa99887766554433221100deadbeefcafebabe0123456789abcdef", + "112233445566778899aabbccddeeff00123456789abcdef00fedcba987654321", + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + ], + "root": "a6ff6556e3c63ba45b8872ec460ac1e8c322574512d94f061e1639e669b74b7e", + "admissionListValid": true + }, + { + "name": "unsorted-asymmetric-byte-values", + "txids": [ + "ffeeddccbbaa99887766554433221100deadbeefcafebabe0123456789abcdef", + "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + ], + "root": "e1c6afbb7257b4a4de469c22d715825b983cbb80b175fa6837244d3b575fbdaa", + "admissionListValid": true + }, + { + "name": "duplicate-looking-three-illustrative", + "txids": [ + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "ffeeddccbbaa99887766554433221100deadbeefcafebabe0123456789abcdef", + "112233445566778899aabbccddeeff00123456789abcdef00fedcba987654321" + ], + "root": "82853ce87598bf885ef5fb2e288d92581a1addc09bc651ea5e3bc2834867f7a0", + "admissionListValid": true, + "illustrativeOnly": true + }, + { + "name": "repeated-last-four-illustrative-invalid", + "txids": [ + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "ffeeddccbbaa99887766554433221100deadbeefcafebabe0123456789abcdef", + "112233445566778899aabbccddeeff00123456789abcdef00fedcba987654321", + "112233445566778899aabbccddeeff00123456789abcdef00fedcba987654321" + ], + "root": "82853ce87598bf885ef5fb2e288d92581a1addc09bc651ea5e3bc2834867f7a0", + "admissionListValid": false, + "illustrativeOnly": true + } + ], + "tac": [ + { + "name": "genesis-height-zero-includes-empty-heights", + "genesisHeight": 0, + "anchors": [ + { + "blockHeight": 0, + "blockHash": "0000000000000000000000000000000000000000000000000000000000000000", + "basmRoot": "0000000000000000000000000000000000000000000000000000000000000000", + "admittedCount": 0, + "rootSource": "empty", + "expectedTac": "3a464e1e43410c7add1dd81c3f10486f41eb473bb43e8d64feca3c7f0c8028d3" + }, + { + "blockHeight": 1, + "blockHash": "1111111111111111111111111111111111111111111111111111111111111111", + "basmRoot": "668066643818b3695ed31c7cfd09dcc8aaeba2f6eb46e6360b041fb3254ccffc", + "admittedCount": 2, + "rootSource": "even-two-asymmetric", + "expectedTac": "3771c04594ce4cf996178d68c3e4d640647b85d008d86a7613ead721c7c64875" + }, + { + "blockHeight": 2, + "blockHash": "2222222222222222222222222222222222222222222222222222222222222222", + "basmRoot": "0000000000000000000000000000000000000000000000000000000000000000", + "admittedCount": 0, + "rootSource": "empty", + "expectedTac": "f5652a4a6f978eac1278b61b7873740ee7109a5f6f05df27032b453eb2687232" + } + ] + }, + { + "name": "nonzero-genesis-mixed-anchors", + "genesisHeight": 100, + "anchors": [ + { + "blockHeight": 100, + "blockHash": "13579bdf2468ace00112233445566778899aabbccddeeff00fedcba987654321", + "basmRoot": "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + "admittedCount": 1, + "rootSource": "singleton-genesis-bitcoin-txid", + "expectedTac": "f5f755f70e8aea69a4f7f5c2d49b782e29de844e2aff24f0089d43d6697655ba" + }, + { + "blockHeight": 101, + "blockHash": "89abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234567", + "basmRoot": "bd162ecb4feb15a6489b8e7a6112dadfb1a699e691077b53dabf4cf79462173c", + "admittedCount": 3, + "rootSource": "tac-three-asymmetric", + "expectedTac": "da1d66659e44f5eb7abb280cf7a323d05ba72b96f78d74de03694bebf3d7af66" + }, + { + "blockHeight": 102, + "blockHash": "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", + "basmRoot": "0000000000000000000000000000000000000000000000000000000000000000", + "admittedCount": 0, + "rootSource": "empty", + "expectedTac": "89f4bd216e66f821ba9cf8da487109efb595f0461a1788dc878040c58a5f2688" + } + ] + } + ] +} diff --git a/packages/overlays/topics/CHANGELOG.md b/packages/overlays/topics/CHANGELOG.md index b47721adb..932e164df 100644 --- a/packages/overlays/topics/CHANGELOG.md +++ b/packages/overlays/topics/CHANGELOG.md @@ -9,6 +9,9 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +- Advances the packed overlay dependency candidate for BASM validation hardening. + Package runtime behavior is unchanged; no consumer migration is required. + ### Added - `tm_uora_dpp` / `ls_uora_dpp`: admission and lookup for UORA attestation diff --git a/packages/overlays/topics/package.json b/packages/overlays/topics/package.json index 0b202c4d8..1f1c0f0ad 100644 --- a/packages/overlays/topics/package.json +++ b/packages/overlays/topics/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-topics", - "version": "1.7.1", + "version": "1.7.2", "sideEffects": false, "engines": { "node": ">=22" From b012b33a6b1891d8635b16bf48c3c83ae441a7fe Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 15:16:51 -0500 Subject: [PATCH 009/168] fix(wallet-toolbox): re-read BHS canonical headers for root checks BHServiceClient no longer caches the queried merkle root. Validity is decided from a freshly read header, so a false result cannot invert on retry and a reorg cannot reuse a stale positive. --- packages/wallet/wallet-toolbox/CHANGELOG.md | 3 +- .../docs/identity-verification.md | 4 +- .../services/chaintracker/BHServiceClient.ts | 13 +-- .../__tests/BHServiceClient.test.ts | 107 ++++++++++++++++++ 4 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/BHServiceClient.test.ts diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index c0a8cf2ff..44efc916e 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -12,7 +12,8 @@ attention to changes that materially alter behavior or extend functionality. envelope, and explicit certificate success. C02/C03 add the exported SDK evidence coordinator, optional chain-provider context markers, bounded transaction/script work sharing, and revalidation of canonical anchors on - positive-cache use. Cached overlay receipts and decrypted certificates remain + positive-cache use. BHServiceClient always re-reads the canonical header for + height-root checks and does not cache a queried root as a reusable positive. Cached overlay receipts and decrypted certificates remain wallet-owned, bounded, and separate from local contacts and trust policy. Existing permission negotiation, pagination, wallet RPC shapes, and resolver call forms remain characterized as compatible. See [identity verification](docs/identity-verification.md) diff --git a/packages/wallet/wallet-toolbox/docs/identity-verification.md b/packages/wallet/wallet-toolbox/docs/identity-verification.md index fc2ddc437..381e8dfe1 100644 --- a/packages/wallet/wallet-toolbox/docs/identity-verification.md +++ b/packages/wallet/wallet-toolbox/docs/identity-verification.md @@ -46,7 +46,9 @@ local-primary uses local identity. An unused provider cannot stand in, and a missing participating identity fails closed. Built-in remote ChainTracks clients advertise reorg-event capability explicitly; method presence is not capability. `Services.getChainTracker()` publishes one coalesced wrapper -before yielding. A remote tip equality read cannot detect an unobserved ABA +before yielding. `BHServiceClient` always re-reads the canonical header for +`isValidRootForHeight` and never treats a previously queried root as a reusable +positive. A remote tip equality read cannot detect an unobserved ABA transition, and a token over multiple sources is not an atomic multi-source snapshot. Canonical roots and observed heights remain dependencies that are rechecked on each use, including positive-cache reuse. Existing trackers may diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts index b5bec6a09..2c442f453 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts @@ -47,13 +47,12 @@ export class BHServiceClient implements ChaintracksServiceClient { } async isValidRootForHeight(root: string, height: number): Promise { - const cachedRoot = this.cache[height] - if (cachedRoot) { - return cachedRoot === root - } - const isValid = await this.bhs.isValidRootForHeight(root, height) - this.cache[height] = root - return isValid + const header = await this.findHeaderForHeight(height) + const merkleRoot = header?.merkleRoot + if (typeof merkleRoot !== 'string' || merkleRoot.length === 0) return false + // Diagnostic only: validity is always decided from a freshly read canonical header. + this.cache[height] = merkleRoot + return merkleRoot === root } async getPresentHeight(): Promise { diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/BHServiceClient.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/BHServiceClient.test.ts new file mode 100644 index 000000000..c3d8e6722 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/BHServiceClient.test.ts @@ -0,0 +1,107 @@ +import { BHServiceClient } from '../BHServiceClient' +import { LocalChainTracker } from '../LocalChainTracker' +import type { ChaintracksClientApi } from '../chaintracks/Api/ChaintracksClientApi' +import { BlockHeader } from '../../../sdk/WalletServices.interfaces' + +const height = 100 +const queried = 'aa'.repeat(32) +const canonical = 'bb'.repeat(32) +const reorged = 'cc'.repeat(32) + +function header(merkleRoot: string): BlockHeader { + return { + version: 1, + previousHash: '00'.repeat(32), + merkleRoot, + time: 1, + bits: 1, + nonce: 1, + height, + hash: 'dd'.repeat(32) + } +} + +function localClient(): ChaintracksClientApi { + return { + getPresentHeight: jest.fn(async () => height), + findChainTipHash: jest.fn(async () => queried), + findHeaderForHeight: jest.fn(async () => header(queried)), + isValidRootForHeight: jest.fn(async () => { + throw new Error('local unavailable') + }), + startListening: jest.fn(async () => undefined), + listening: jest.fn(async () => undefined) + } as unknown as ChaintracksClientApi +} + +function bhsClient(): BHServiceClient { + const client = new BHServiceClient('main', 'https://headers.example', 'test-key') + jest.spyOn(client.bhs, 'isValidRootForHeight') + return client +} + +describe('BHServiceClient height-root cache', () => { + test('does not invert a false result into a cached positive for the queried root', async () => { + const client = bhsClient() + jest.spyOn(client, 'findHeaderForHeight').mockResolvedValue(header(canonical)) + ;(client.bhs.isValidRootForHeight as jest.Mock).mockResolvedValue(false) + + await expect(client.isValidRootForHeight(queried, height)).resolves.toBe(false) + await expect(client.isValidRootForHeight(queried, height)).resolves.toBe(false) + expect(client.findHeaderForHeight).toHaveBeenCalledTimes(2) + expect(client.cache[height]).toBe(canonical) + }) + + test('re-reads a later different root at the same height', async () => { + const client = bhsClient() + jest + .spyOn(client, 'findHeaderForHeight') + .mockResolvedValueOnce(header(queried)) + .mockResolvedValueOnce(header(canonical)) + ;(client.bhs.isValidRootForHeight as jest.Mock).mockResolvedValue(true) + + await expect(client.isValidRootForHeight(queried, height)).resolves.toBe(true) + await expect(client.isValidRootForHeight(canonical, height)).resolves.toBe(true) + expect(client.findHeaderForHeight).toHaveBeenCalledTimes(2) + expect(client.cache[height]).toBe(canonical) + }) + + test('does not reuse a positive after the canonical header at that height changes', async () => { + const client = bhsClient() + jest + .spyOn(client, 'findHeaderForHeight') + .mockResolvedValueOnce(header(queried)) + .mockResolvedValueOnce(header(reorged)) + ;(client.bhs.isValidRootForHeight as jest.Mock).mockResolvedValueOnce(true).mockResolvedValueOnce(false) + + await expect(client.isValidRootForHeight(queried, height)).resolves.toBe(true) + await expect(client.isValidRootForHeight(queried, height)).resolves.toBe(false) + expect(client.findHeaderForHeight).toHaveBeenCalledTimes(2) + expect(client.cache[height]).toBe(reorged) + }) + + test('does not cache a queried root when the canonical header is missing', async () => { + const client = bhsClient() + jest.spyOn(client, 'findHeaderForHeight').mockResolvedValue(undefined) + ;(client.bhs.isValidRootForHeight as jest.Mock).mockResolvedValue(false) + + await expect(client.isValidRootForHeight(queried, height)).resolves.toBe(false) + expect(client.cache[height]).toBeUndefined() + }) + + test('LocalChainTracker fallback re-reads BHServiceClient roots instead of the queried-root cache', async () => { + const fallback = bhsClient() + jest.spyOn(fallback, 'findHeaderForHeight').mockResolvedValue(header(canonical)) + ;(fallback.bhs.isValidRootForHeight as jest.Mock).mockResolvedValue(false) + const tracker = new LocalChainTracker({ + local: localClient(), + fallbacks: [fallback], + mode: 'remote-only' + }) + + await expect(tracker.isValidRootForHeight(queried, height)).resolves.toBe(false) + await expect(tracker.isValidRootForHeight(queried, height)).resolves.toBe(false) + expect(fallback.findHeaderForHeight).toHaveBeenCalledTimes(2) + expect(fallback.cache[height]).toBe(canonical) + }) +}) From e82b77eeabfaf78e1a5b60f3367e56dc00fa828b Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 15:20:38 -0500 Subject: [PATCH 010/168] fix(sdk): keep overlay host cache and query() deadline honest Store discovery bounds on the SLAP host cache so a tighter-limit query cannot freeze a truncated host set for a later broader attempt. Throw the historical no-competent-hosts error from query()/queryDetailed when a deadline expires before any host is admitted; query$ still emits the deadline snapshot. --- packages/sdk/CHANGELOG.md | 3 + .../sdk/src/overlay-tools/LookupResolver.ts | 160 +++++++++++++++--- .../__tests/LookupResolver.dynamic.test.ts | 101 +++++++++++ 3 files changed, 240 insertions(+), 24 deletions(-) diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 4f913e481..b1169f4d2 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -254,6 +254,9 @@ All notable changes to this project will be documented in this file. The format ### Changed +- LookupResolver host cache no longer lets a tighter-limit discovery satisfy a + later larger query, and `query()` still throws the historical no-competent-hosts + error when a deadline expires before any host is admitted. - Batch BEEF mutation bookkeeping and reuse compound Merkle intermediate hashes. The optional asynchronous P2PKH backend now forwards its already validated compressed public key directly into the unlocking script. Existing BEEF diff --git a/packages/sdk/src/overlay-tools/LookupResolver.ts b/packages/sdk/src/overlay-tools/LookupResolver.ts index c0e674110..e38f6338b 100644 --- a/packages/sdk/src/overlay-tools/LookupResolver.ts +++ b/packages/sdk/src/overlay-tools/LookupResolver.ts @@ -352,6 +352,66 @@ interface CacheOptions { txMemoTtlMs?: number } +/** Discovery fields that can truncate the cached SLAP host set. */ +interface LookupDiscoveryBound { + maxHosts: number + maxHostsPerTracker: number + maxTrackers: number + maxResponseBytes: number + maxTotalBytes: number + maxOutputs: number +} + +interface LookupHostsCacheEntry extends LookupDiscoveryBound { + hosts: string[] + expiresAt: number + discoveryComplete?: boolean + trackersFailed?: number + limitsHit?: string[] +} + +function lookupDiscoveryBound(limits: LookupLimits): LookupDiscoveryBound { + return { + maxHosts: limits.maxHosts, + maxHostsPerTracker: limits.maxHostsPerTracker, + maxTrackers: limits.maxTrackers, + maxResponseBytes: limits.maxResponseBytes, + maxTotalBytes: limits.maxTotalBytes, + maxOutputs: limits.maxOutputs + } +} + +/** In-flight discovery identity: service plus the limits that shape tracker work. */ +function lookupDiscoveryCacheKey(service: string, limits: LookupLimits): string { + const bound = lookupDiscoveryBound(limits) + return JSON.stringify([ + service, + bound.maxHosts, + bound.maxHostsPerTracker, + bound.maxTrackers, + limits.trackerConcurrency, + bound.maxResponseBytes, + bound.maxTotalBytes, + bound.maxOutputs + ]) +} + +/** True when `cached` was produced with bounds at least as permissive as `needed`. */ +function lookupDiscoveryCovers( + cached: Partial | undefined, + needed: LookupDiscoveryBound +): boolean { + if (cached === undefined) return false + return ( + (cached.maxHosts ?? 0) >= needed.maxHosts && + (cached.maxHostsPerTracker ?? 0) >= needed.maxHostsPerTracker && + (cached.maxTrackers ?? 0) >= needed.maxTrackers && + (cached.maxResponseBytes ?? 0) >= needed.maxResponseBytes && + (cached.maxTotalBytes ?? 0) >= needed.maxTotalBytes && + (cached.maxOutputs ?? 0) >= needed.maxOutputs + ) +} + /** Configuration options for the Lookup resolver. */ export interface LookupResolverConfig { /** Defaults for the bounded discovery, scheduler and receipt intake. */ @@ -777,7 +837,7 @@ export default class LookupResolver { private readonly telemetry: Telemetry // ---- Caches / memoization ---- - private readonly hostsCache: Map + private readonly hostsCache: Map private readonly hostsInFlight: Map private readonly limits: LookupLimits private activeQueries = 0 @@ -891,25 +951,34 @@ export default class LookupResolver { } finally { await iter.return?.(undefined) } + const progress: LookupAnswerProgress = last ?? { + type: 'output-list', + outputs: [], + txIds: [], + isFinal: true, + hostCount: 0, + completedHosts: 0, + successfulHosts: 0, + emptyHosts: 0, + failedHosts: 0, + rejectedHosts: 0, + freeformHosts: 0, + terminalReason: 'settled', + ...(options?.correlationId !== undefined ? { correlationId: options.correlationId } : {}) + } + // Promise callers cannot see terminalReason. A deadline that admitted no + // host is a miss, not a successful empty answer from a queried host. + if (progress.hostCount === 0 && progress.terminalReason !== 'cancelled') { + throw new Error( + `No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${question.service}` + ) + } return { answer: { type: 'output-list', - outputs: last?.outputs ?? [] + outputs: progress.outputs }, - progress: last ?? { - type: 'output-list', - outputs: [], - txIds: [], - isFinal: true, - hostCount: 0, - completedHosts: 0, - successfulHosts: 0, - emptyHosts: 0, - failedHosts: 0, - rejectedHosts: 0, - freeformHosts: 0, - ...(options?.correlationId !== undefined ? { correlationId: options.correlationId } : {}) - } + progress } } @@ -1233,14 +1302,23 @@ export default class LookupResolver { } else { const cached = this.hostsCache.get(question.service) const configuredAdditional = this.additionalHosts[question.service] ?? [] - const cacheAvailable = cached?.hosts.some(host => (this.hostReputation.snapshot(host)?.backoffUntil ?? 0) <= Date.now()) ?? false - const key = JSON.stringify([question.service, limits.maxHosts, limits.maxHostsPerTracker, - limits.maxTrackers, limits.trackerConcurrency, limits.maxResponseBytes, limits.maxTotalBytes, limits.maxOutputs]) + const cacheHasAvailableHost = + cached?.hosts.some(host => (this.hostReputation.snapshot(host)?.backoffUntil ?? 0) <= Date.now()) ?? + false + const cacheCoversCaller = lookupDiscoveryCovers(cached, limits) + const cacheFresh = cached !== undefined && cached.expiresAt > Date.now() + const key = lookupDiscoveryCacheKey(question.service, limits) let discovery = this.hostsInFlight.get(key) - const refresh = discovery !== undefined || cached === undefined || cached.expiresAt <= Date.now() || !cacheAvailable - const initialSources = Number(cached !== undefined && cacheAvailable) + Number(configuredAdditional.length > 0) + const refresh = + discovery !== undefined || + cached === undefined || + !cacheCoversCaller || + !cacheFresh || + !cacheHasAvailableHost + const initialSources = + Number(cached !== undefined && cacheHasAvailableHost) + Number(configuredAdditional.length > 0) const initialQuota = refresh ? Math.max(1, Math.floor(limits.maxHosts / (initialSources + Math.max(1, Math.min(this.slapTrackers.length, limits.maxTrackers))))) : limits.maxHosts - if (cached !== undefined && cacheAvailable) { + if (cached !== undefined && cacheHasAvailableHost) { // Reserve a source share for cached membership and each late tracker. const cachedLimit = initialQuota admit('cache', cached.hosts.slice(0, cachedLimit)) @@ -1275,8 +1353,7 @@ export default class LookupResolver { this.hostsInFlight.delete(key) if (abandoned) return const hosts = Array.from(new Set(Array.from(state.sources.values()).flat())).slice(0, limits.maxHosts) - if (!this.hostsCache.has(question.service) && this.hostsCache.size >= this.hostsMaxEntries) this.evictOldest(this.hostsCache) - this.hostsCache.set(question.service, { hosts, expiresAt: Date.now() + this.hostsTtlMs, discoveryComplete: state.trackersFailed === 0 && state.limitsHit.size === 0 && state.skippedHosts === 0, trackersFailed: state.trackersFailed, limitsHit: Array.from(state.limitsHit) }) + this.rememberDiscoveredHosts(question.service, hosts, limits, state) }) if (this.slapTrackers.length > limits.maxTrackers) discovery.state.limitsHit.add('maxTrackers') if (normalized.length !== this.slapTrackers.length || trackers.length < Math.min(normalized.length, limits.maxTrackers)) { @@ -1367,6 +1444,41 @@ export default class LookupResolver { if (firstKey !== undefined) m.delete(firstKey) } + /** + * Remember SLAP hosts for a service. A tighter-limit discovery must not + * replace a still-fresh broader cache, and a later broader query must not + * treat a truncated entry as complete. + */ + private rememberDiscoveredHosts( + service: string, + hosts: string[], + limits: LookupLimits, + state: LookupDiscoveryUpdate + ): void { + const existing = this.hostsCache.get(service) + const now = Date.now() + if ( + existing !== undefined && + existing.expiresAt > now && + lookupDiscoveryCovers(existing, limits) && + !lookupDiscoveryCovers(lookupDiscoveryBound(limits), existing) + ) { + return + } + if (existing === undefined && this.hostsCache.size >= this.hostsMaxEntries) { + this.evictOldest(this.hostsCache) + } + this.hostsCache.set(service, { + ...lookupDiscoveryBound(limits), + hosts, + expiresAt: now + this.hostsTtlMs, + discoveryComplete: + state.trackersFailed === 0 && state.limitsHit.size === 0 && state.skippedHosts === 0, + trackersFailed: state.trackersFailed, + limitsHit: Array.from(state.limitsHit) + }) + } + private assertValidOverrideServices(overrides: Record): void { for (const service of Object.keys(overrides)) { if (!service.startsWith('ls_')) { diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts index a8a78f552..760b1ecd9 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts @@ -838,4 +838,105 @@ describe('LookupResolver dynamic discovery', () => { outputs: [{ beef: makeBeef(91), outputIndex: 0 }] }) }) + + it('does not let a tighter-limit cache hide later tracker hosts from a subsequent default query', async () => { + const tracker = 'https://tight-cache-tracker.example' + const firstHost = 'https://tight-cache-a.example' + const laterHost = 'https://tight-cache-b.example' + const service = 'ls_tight_cache' + const firstReceipt = await slapReceipt(170, firstHost, service) + const laterReceipt = await slapReceipt(171, laterHost, service) + const firstBeef = makeBeef(101) + const laterBeef = makeBeef(102) + const lookup = jest.fn(async (url: string) => { + if (url === tracker) { + return { type: 'output-list' as const, outputs: [firstReceipt, laterReceipt] } + } + return { + type: 'output-list' as const, + outputs: [{ beef: url === firstHost ? firstBeef : laterBeef, outputIndex: 0 }] + } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + + const tight = resolver.query({ service, query: { n: 1 } }, undefined, { limits: { maxHosts: 1 } }) + await jest.runAllTimersAsync() + await tight + + const hostCallsAfterTight = lookup.mock.calls + .map(([url]) => url) + .filter((url: string) => url === firstHost || url === laterHost) + expect(hostCallsAfterTight).toHaveLength(1) + + const full = resolver.query({ service, query: { n: 2 } }) + await jest.runAllTimersAsync() + const answer = await full + expect(answer.outputs).toEqual( + expect.arrayContaining([ + { beef: firstBeef, outputIndex: 0 }, + { beef: laterBeef, outputIndex: 0 } + ]) + ) + expect(answer.outputs).toHaveLength(2) + expect(lookup.mock.calls.filter(([url]) => url === tracker).length).toBeGreaterThan(1) + expect(lookup.mock.calls.map(([url]) => url)).toEqual( + expect.arrayContaining([firstHost, laterHost]) + ) + }) + + it('throws from query() when a deadline expires before any host is admitted', async () => { + const tracker = 'https://deadline-miss-tracker.example' + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + slapTrackers: [tracker] + }) + const pending = expect( + resolver.query({ service: 'ls_deadline_miss', query: {} }, undefined, { + deadlineMs: 25 + }) + ).rejects.toThrow( + 'No competent mainnet hosts found by the SLAP trackers for lookup service: ls_deadline_miss' + ) + await jest.advanceTimersByTimeAsync(25) + await pending + }) + + it('keeps query$ deadline snapshots when no host was admitted while Promise callers still throw', async () => { + const tracker = 'https://deadline-snapshot-tracker.example' + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + slapTrackers: [tracker] + }) + const received: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$( + { service: 'ls_deadline_snapshot', query: {} }, + undefined, + { deadlineMs: 25 } + )) { + received.push(item) + } + })() + await jest.advanceTimersByTimeAsync(25) + await pending + expect(received).toHaveLength(1) + expect(received[0]).toMatchObject({ + isFinal: true, + terminalReason: 'deadline', + hostCount: 0, + outputs: [] + }) + }) }) From fc83b6bb407b0f51720536d879500c6c11e92b88 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 15:28:47 -0500 Subject: [PATCH 011/168] fix(overlay): compare Mongo uint32 indexes as integers Collection validators used lexicographic $lte on unpadded outputIndex strings, which rejected legal indexes such as "9". Bound uint32 fields with $toLong on outputs, consumption edges, GASP nodes, and SHIP/SLAP. --- .../src/__tests/mongo/MongoSchema.test.ts | 117 ++++++++++++++++++ .../overlay/src/storage/mongo/MongoSchema.ts | 31 +++-- specs/overlay/mongo-v1.md | 4 +- 3 files changed, 135 insertions(+), 17 deletions(-) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts index 1e5a965e0..aa09e6b1c 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts @@ -45,10 +45,31 @@ describe('Mongo schema codecs', () => { }) test('keeps output indexes in the wire uint32 domain', () => { + expect(encodeMongoOutputIndex('9')).toBe('9') + expect(encodeMongoOutputIndex('50')).toBe('50') expect(encodeMongoOutputIndex('4294967295')).toBe('4294967295') expect(() => encodeMongoOutputIndex('4294967296')).toThrow('Invalid storage output index') }) + test('compares uint32 collection bounds as integers, not lexicographic strings', () => { + const names = [ + MongoCollectionNames.outputs, + MongoCollectionNames.consumptionEdges, + MongoCollectionNames.gaspNodes, + MongoCollectionNames.shipRecords, + MongoCollectionNames.slapRecords + ] + for (const name of names) { + const definition = MongoCollectionDefinitions.find(item => item.name === name) + if (definition === undefined) throw new Error(`Missing schema definition for ${name}`) + const encoded = JSON.stringify(definition.validator) + expect(encoded).toContain('$toLong') + expect(encoded).not.toContain('"$outputIndex","4294967295"') + expect(encoded).not.toContain('"$sourceOutputIndex","4294967295"') + expect(encoded).not.toContain('"$consumerOutputIndex","4294967295"') + } + }) + test('defines every Overlay-owned collection with strict versioned validators', () => { expect(MongoCollectionDefinitions.map(definition => definition.name)).toEqual( Object.values(MongoCollectionNames) @@ -176,6 +197,102 @@ describe('Mongo schema bootstrap', () => { ).rejects.toThrow() }) + test('accepts canonical uint32 output indexes 9 and 4294967295 and rejects 4294967296', async () => { + const now = new Date() + const base = { schemaVersion: 1, ...fixture.scope, createdAt: now, updatedAt: now } + const txid = 'd1'.repeat(32) + const output = (id: string, outputIndex: string) => ({ + _id: id, + ...base, + topic: 'tm_uint32', + txid, + outputIndex, + satoshis: encodeMongoUint64('1'), + score: encodeMongoUint64('0'), + scriptPayloadId: 'script', + scriptOffset: encodeMongoUint64('0'), + scriptByteLength: encodeMongoUint64('0'), + state: 'unspent', + version: 'v1' + }) + const edge = ( + id: string, + sourceOutputIndex: string, + consumerOutputIndex: string, + consumerTxid: string + ) => ({ + _id: id, + ...base, + topic: 'tm_uint32', + sourceTxid: txid, + sourceOutputIndex, + consumerTxid, + consumerOutputIndex + }) + await fixture.db + .collection(MongoCollectionNames.outputs) + .insertMany([output('output-index-9', '9'), output('output-index-max', '4294967295')]) + await expect( + fixture.db + .collection(MongoCollectionNames.outputs) + .insertOne(output('output-index-overflow', '4294967296')) + ).rejects.toThrow() + await fixture.db.collection(MongoCollectionNames.consumptionEdges).insertMany([ + edge('edge-index-9', '9', '50', 'd2'.repeat(32)), + edge('edge-index-max', '4294967295', '9', 'd3'.repeat(32)) + ]) + await expect( + fixture.db + .collection(MongoCollectionNames.consumptionEdges) + .insertOne(edge('edge-index-overflow', '4294967296', '0', 'd4'.repeat(32))) + ).rejects.toThrow() + await fixture.db.collection(MongoCollectionNames.gaspNodes).insertOne({ + _id: 'gasp-index-9', + ...base, + graphId: 'graph-uint32', + txid, + outputIndex: '9', + state: 'receiving' + }) + await expect( + fixture.db.collection(MongoCollectionNames.gaspNodes).insertOne({ + _id: 'gasp-index-overflow', + ...base, + graphId: 'graph-uint32-overflow', + txid, + outputIndex: '4294967296', + state: 'receiving' + }) + ).rejects.toThrow() + for (const name of [MongoCollectionNames.shipRecords, MongoCollectionNames.slapRecords]) { + await fixture.db.collection(name).insertOne({ + _id: `${name}-index-9`, + ...base, + txid, + outputIndex: '9', + domain: 'example.com', + state: 'active' + }) + await expect( + fixture.db.collection(name).insertOne({ + _id: `${name}-index-overflow`, + ...base, + txid: 'd5'.repeat(32), + outputIndex: '4294967296', + domain: 'example.com', + state: 'active' + }) + ).rejects.toThrow() + } + const ledger = await fixture.db + .collection(MongoCollectionNames.schema) + .findOne({ _id: mongoNodeKey(fixture.scope) }) + expect(ledger?.schemaFingerprint).toMatch(/^[0-9a-f]{64}$/) + expect(JSON.stringify(MongoCollectionDefinitions.map(item => item.validator))).toContain( + '$toLong' + ) + }) + test('rejects direct documents without required audit dates', async () => { await expect( fixture.db.collection(MongoCollectionNames.payloads).insertOne({ diff --git a/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts b/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts index 91cd7d965..bff77393e 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts @@ -44,6 +44,7 @@ export const MongoGridFsBucketName = 'overlayPayloads' const schemaVersion = 1 const maxUint64 = '18446744073709551615' +const maxUint32 = 4294967295 const paddedUint64Pattern = '^[0-9]{20}$' const hashPattern = '^[0-9a-f]{64}$' const uuidPattern = '^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$' @@ -169,6 +170,15 @@ function scopedValidator( .map(([field]) => ({ $or: [{ $eq: [{ $type: `$${field}` }, 'missing'] }, { $lte: [`$${field}`, maxUint64] }] })) + // Unpadded canonical uint32 strings are not lexicographically ordered; compare as integers. + const uint32Bounds = Object.entries(fields) + .filter(([, definition]) => definition === uint32) + .map(([field]) => ({ + $or: [ + { $eq: [{ $type: `$${field}` }, 'missing'] }, + { $lte: [{ $toLong: `$${field}` }, maxUint32] } + ] + })) return { $and: [ { @@ -187,6 +197,7 @@ function scopedValidator( } }, ...uint64Bounds.map(bound => ({ $expr: bound })), + ...uint32Bounds.map(bound => ({ $expr: bound })), ...(options.extra === undefined ? [] : [options.extra]) ] } @@ -434,7 +445,7 @@ const definitions: MongoCollectionDefinition[] = [ 'state', 'version' ], - { node: true, extra: { $expr: { $lte: ['$outputIndex', '4294967295'] } } } + { node: true } ), indexes: [ index( @@ -459,17 +470,7 @@ const definitions: MongoCollectionDefinition[] = [ consumerOutputIndex: uint32 }, ['topic', 'sourceTxid', 'sourceOutputIndex', 'consumerTxid', 'consumerOutputIndex'], - { - node: true, - extra: { - $expr: { - $and: [ - { $lte: ['$sourceOutputIndex', '4294967295'] }, - { $lte: ['$consumerOutputIndex', '4294967295'] } - ] - } - } - } + { node: true } ), indexes: [ index( @@ -711,8 +712,7 @@ stateDefinition( { unique: true } ), index({ graphId: 1 }, 'graph_lookup') - ], - { $expr: { $lte: ['$outputIndex', '4294967295'] } } + ] ) stateDefinition( MongoCollectionNames.manifestComponents, @@ -770,8 +770,7 @@ for (const name of [MongoCollectionNames.shipRecords, MongoCollectionNames.slapR { unique: true } ), index({ domain: 1, topic: 1, createdAt: 1 }, 'discovery_lookup') - ], - { $expr: { $lte: ['$outputIndex', '4294967295'] } } + ] ) } stateDefinition( diff --git a/specs/overlay/mongo-v1.md b/specs/overlay/mongo-v1.md index 5bfb02a43..93d0a0f32 100644 --- a/specs/overlay/mongo-v1.md +++ b/specs/overlay/mongo-v1.md @@ -46,7 +46,9 @@ Chain-wide records include `network` and `genesisHash`; node-owned records add delimiter-joined string. Exact wire integers are canonical decimal strings: the schema helpers reject non-canonical values and preserve uint64 values without passing them through an unsafe JavaScript number. Output indices have -the stricter uint32 range. Collection validators and indexes bound row shape, +the stricter uint32 range and are stored as unpadded canonical decimals; +collection validators compare them as integers (`$toLong`), not as +lexicographic strings. Collection validators and indexes bound row shape, field length, array size, receipt size, and indexed identity fields so an unbounded owner record is not silently created. MongoDB's BSON document ceiling is 16 MiB; schema bounds remain necessary even below that ceiling. See From 752464dff9d91ee4d0ee4abdd1ad6997a4141912 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 15:40:03 -0500 Subject: [PATCH 012/168] fix(overlay): bind BASM admitted indices without coinbase maturity Verify claimed admitted-list block indices against the compound Merkle path even when every remote txid is already local. Check inclusion with the chain-tracker root at the claimed height instead of MerklePath.verify, which also enforces coinbase 100-block spendability. --- docs/reference/package-api-migrations.md | 6 +- governance/package-release-notes.json | 4 +- packages/overlays/overlay/CHANGELOG.md | 9 ++- packages/overlays/overlay/README.md | 8 +- .../overlays/overlay/docs/BRC-136-BASM.md | 5 +- packages/overlays/overlay/src/Engine.ts | 22 +++-- .../src/__tests/BASMReconciliation.test.ts | 81 ++++++++++++++++++- 7 files changed, 116 insertions(+), 19 deletions(-) diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index a1b4b94e4..1af096bb6 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -42,7 +42,7 @@ and clean-consumer tests remain the executable type authority. | `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | | `@bsv/message-box-client` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | -| `@bsv/overlay` | `2.2.1` | `2.4.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. Existing lookup limits and abort-hook guidance remain applicable. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| `@bsv/overlay` | `2.2.1` | `2.4.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. Existing lookup limits and abort-hook guidance remain applicable. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.2` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update. | | `@bsv/overlay-express` | `2.5.0` | `2.7.0` | minor | [API and usage](../packages/overlays/overlay-express.md) | Valid legacy JSON shapes, numeric-string heights, public CORS, topic-free raw transactions, configured limits and synchronization defaults remain supported. No storage migration is required. Malformed/coercive inputs are rejected and missing BASM engine/storage capabilities return HTTP 400 with code BASM_UNSUPPORTED. Existing header providers remain valid without a blockTransactionCount field. TTN setup and license-notice requirements remain unchanged. | | `@bsv/overlay-topics` | `1.6.10` | `1.7.2` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. No migration is required for this dependency candidate update. | @@ -277,8 +277,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay.md](../packages/overlays/overlay.md) - Source: [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) -- Release note: Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Retains the engine lookup cardinality ceiling, optional TopicManager abort hook and scoped pre-uniformization Open BSV License version 4 continuity notice. -- Migration: Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. Existing lookup limits and abort-hook guidance remain applicable. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. +- Release note: Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Claimed admitted-list indices are bound to the compound path even when every remote txid is already local, and inclusion uses chain-tracker root/height rather than coinbase maturity. Retains the engine lookup cardinality ceiling, optional TopicManager abort hook and scoped pre-uniformization Open BSV License version 4 continuity notice. +- Migration: Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. Existing lookup limits and abort-hook guidance remain applicable. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index b4dba89e8..65e51f4b6 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -126,8 +126,8 @@ "name": "@bsv/overlay", "publishedVersion": "2.2.1", "releaseType": "minor", - "summary": "Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Retains the engine lookup cardinality ceiling, optional TopicManager abort hook and scoped pre-uniformization Open BSV License version 4 continuity notice.", - "migration": "Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. Existing lookup limits and abort-hook guidance remain applicable. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package." + "summary": "Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Claimed admitted-list indices are bound to the compound path even when every remote txid is already local, and inclusion uses chain-tracker root/height rather than coinbase maturity. Retains the engine lookup cardinality ceiling, optional TopicManager abort hook and scoped pre-uniformization Open BSV License version 4 continuity notice.", + "migration": "Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. Existing lookup limits and abort-hook guidance remain applicable. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package." }, { "name": "@bsv/overlay-discovery-services", diff --git a/packages/overlays/overlay/CHANGELOG.md b/packages/overlays/overlay/CHANGELOG.md index bf11cf34f..debff5b60 100644 --- a/packages/overlays/overlay/CHANGELOG.md +++ b/packages/overlays/overlay/CHANGELOG.md @@ -23,9 +23,12 @@ All notable changes to this project will be documented in this file. The format - Added bounded BASM JSON peer validation, classified capability/resource errors, canonical header and optional full-block-count position checks, and explicit position assurance in sync reports. Fixed default forward pages to fit the - public server's 1,000-anchor limit. Historical sync refuses an untrusted TAC - prefix; durable bootstrap, chunked recovery and topic status remain pending. - No storage migration or automatic sync activation is included. + public server's 1,000-anchor limit. Claimed admitted-list indices are bound to + the compound path even when every remote txid is already local. Inclusion uses + chain-tracker root/height rather than coinbase maturity. Historical sync + refuses an untrusted TAC prefix; durable bootstrap, chunked recovery and topic + status remain pending. No storage migration or automatic sync activation is + included. ### Added - (Include new features or significant user-visible enhancements here.) diff --git a/packages/overlays/overlay/README.md b/packages/overlays/overlay/README.md index 540110cc7..aea9ca272 100644 --- a/packages/overlays/overlay/README.md +++ b/packages/overlays/overlay/README.md @@ -152,8 +152,12 @@ field, and the engine does not download full blocks to infer it. Forward sync pages now contain at most 1,000 anchors to fit the standard HTTP server. Proof height, requested original index, canonical hash/root, raw byte identity, TAC continuity, and repeated peer anchors are checked before historical -submission. Historical mode still applies the local TopicManager and suppresses -broadcast and propagation. Automatic BASM sync remains disabled by default. +submission. Claimed admitted-list indices are bound to the compound path whenever +a remote list is used as evidence, including when every remote txid is already +local. Inclusion uses the chain tracker root/height check rather than +`MerklePath.verify`, which also enforces coinbase 100-block spendability. +Historical mode still applies the local TopicManager and suppresses broadcast +and propagation. Automatic BASM sync remains disabled by default. This is bounded protocol hardening, not durable recovery. An empty local node whose topic genesis precedes the recent bootstrap window now refuses the diff --git a/packages/overlays/overlay/docs/BRC-136-BASM.md b/packages/overlays/overlay/docs/BRC-136-BASM.md index 59e17bd25..649a75834 100644 --- a/packages/overlays/overlay/docs/BRC-136-BASM.md +++ b/packages/overlays/overlay/docs/BRC-136-BASM.md @@ -13,7 +13,10 @@ performs bounded forward work and refuses historical divergence or an untrusted bootstrap prefix. See the [current validation and recovery limits](../README.md#basm-peer-validation-and-current-recovery-limits) before interpreting a sync report. Optional canonical full-block counts provide stronger original-position evidence; header-only providers retain explicitly -weaker evidence. Peer anchor equality is not durable or global completeness. +weaker evidence. A remote admitted list is bound to its compound Merkle path +even when every claimed txid is already local. Inclusion is canonical +root/height, not coinbase maturity. Peer anchor equality is not durable or +global completeness. ## What BRC-136 gives you diff --git a/packages/overlays/overlay/src/Engine.ts b/packages/overlays/overlay/src/Engine.ts index 1bc434650..6e88455ed 100644 --- a/packages/overlays/overlay/src/Engine.ts +++ b/packages/overlays/overlay/src/Engine.ts @@ -1724,6 +1724,7 @@ export class Engine { if (toHeight === remoteTip.blockHeight) requireBASM(previousTac === remoteTip.tac, 'BASM range differs from its tip') for (const remoteAnchor of range.anchors) { await this.reconcileRemoteAnchor(topic, remote, remoteAnchor, report) + if (report.status === 'diverged') return report } const finalRemoteTip = await remote.requestTopicAnchorTip() @@ -1803,13 +1804,14 @@ export class Engine { .filter(txid => !localTxids.has(txid)) report.missingTxids.push(...missingTxids) + // The BASM root only commits to txid order. Bind claimed original indices to + // the compound path even when every remote txid is already local. + const assurance = await this.fetchBASMMissingTransactions(remote, topic, remoteAnchor, admittedResponse.admitted, missingTxids) + if (report.positionValidation !== 'encoded-offset-only') report.positionValidation = assurance if (missingTxids.length === 0) { report.status = 'diverged' return } - - const assurance = await this.fetchBASMMissingTransactions(remote, topic, remoteAnchor, admittedResponse.admitted, missingTxids) - if (report.positionValidation !== 'encoded-offset-only') report.positionValidation = assurance report.fetchedTxCount += missingTxids.length } @@ -1839,10 +1841,16 @@ export class Engine { const leaf = compoundPath.path[0]?.find(item => item.hash === txid) requireBASM(leaf !== undefined && leaf.offset === blockIndex, 'BASM proof does not bind the admitted block index') requireBASM(compoundPath.path[0].length !== 1 || compoundPath.path.length !== 1 || blockIndex === 0, 'BASM singleton proof has a nonzero block index') - const valid = await compoundPath.verify(txid, this.chainTracker) - if (!valid) { - throw new Error(`Peer supplied invalid compound Merkle path for ${txid} at height ${anchor.blockHeight}`) - } + requireBASM(compoundPath.computeRoot(txid) === proofRoot, 'BASM proof root does not match the admitted transaction') + } + // Inclusion is root/height, not coinbase maturity. MerklePath.verify also + // applies the 100-block spendability rule at offset 0. + const valid = await this.chainTracker.isValidRootForHeight(proofRoot, compoundPath.blockHeight) + if (!valid) { + throw new Error(`Peer supplied invalid compound Merkle path at height ${anchor.blockHeight}`) + } + if (txids.length === 0) { + return proofHeader.blockTransactionCount === undefined ? 'encoded-offset-only' : 'canonical-count' } const rawResponse = await remote.requestRawTransactions(txids) diff --git a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts index 9545472e2..4b04d4df1 100644 --- a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts +++ b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts @@ -94,6 +94,7 @@ function fixture() { transactions, ids, path, + root, submit, storage, tracker, @@ -119,7 +120,8 @@ describe('BASM reconciliation evidence binding', () => { expect(callback).toBeUndefined() expect(mode).toBe('historical-tx') } - expect(f.tracker.isValidRootForHeight).toHaveBeenCalledTimes(2) + expect(f.tracker.isValidRootForHeight).toHaveBeenCalledTimes(1) + expect(f.tracker.isValidRootForHeight).toHaveBeenCalledWith(f.root, blockHeight) }) it('reports stronger position validation only when a trusted count is bound to the canonical header', async () => { @@ -338,6 +340,83 @@ describe('BASM reconciliation evidence binding', () => { expect(f.submit).not.toHaveBeenCalled() }) + it('binds claimed indices to the compound path when every remote txid is already local', async () => { + const f = fixture() + f.storage.findAdmittedTransactionsForBlock.mockResolvedValue([...f.admitted]) + f.admitted[0].blockIndex = 0 + f.admitted[1].blockIndex = 2 + f.anchor.basmRoot = computeBasmRoot(f.admitted) + f.anchor.tac = computeTac(BASM_ZERO_HASH, blockHash, f.anchor.basmRoot) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.message).toContain('admitted block index') + expect(f.requests.some(request => request.path === '/requestCompoundMerklePath')).toBe(true) + expect(f.requests.some(request => request.path === '/requestRawTransactions')).toBe(false) + expect(f.submit).not.toHaveBeenCalled() + }) + + it('still requests a compound path before reporting local-superset divergence', async () => { + const f = fixture() + f.storage.findAdmittedTransactionsForBlock.mockResolvedValue([...f.admitted]) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('diverged') + expect(report.positionValidation).toBe('encoded-offset-only') + expect(f.requests.some(request => request.path === '/requestCompoundMerklePath')).toBe(true) + expect(f.requests.some(request => request.path === '/requestRawTransactions')).toBe(false) + expect(f.submit).not.toHaveBeenCalled() + }) + + it('accepts an admitted coinbase in a block younger than 100 confirmations', async () => { + const f = fixture() + const coinbase = new Transaction( + 1, + [], + [{ satoshis: 50, lockingScript: LockingScript.fromASM('OP_TRUE') }], + 0 + ) + const txid = coinbase.id('hex') + const path = new MerklePath(blockHeight, [[{ offset: 0, hash: txid, txid: true }]]) + const admitted = [{ txid, blockIndex: 0 }] + const anchor: TopicBlockAnchor = { + topic, + blockHeight, + blockHash, + basmRoot: txid, + admittedCount: 1, + tac: computeTac(BASM_ZERO_HASH, blockHash, txid) + } + f.tracker.currentHeight.mockResolvedValue(blockHeight + 50) + f.tracker.isValidRootForHeight.mockImplementation( + async (candidate: string, height: number) => candidate === txid && height === blockHeight + ) + f.engine.topicAnchorHeaderResolver = jest.fn(async height => ({ + blockHeight: height, + blockHash, + merkleRoot: txid + })) + f.responses['/requestTopicAnchorTip'] = anchor + f.responses['/requestTopicAnchorRange'] = { topic, anchors: [anchor] } + f.responses['/requestAdmittedList'] = { topic, blockHeight, blockHash, admitted } + f.responses['/requestCompoundMerklePath'] = { + topic, + blockHeight, + txids: [txid], + merklePath: path.toHex() + } + f.responses['/requestRawTransactions'] = { + transactions: [{ txid, rawTx: coinbase.toHex() }], + missing: [] + } + const verify = jest.spyOn(MerklePath.prototype, 'verify') + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('advanced') + expect(report.fetchedTxCount).toBe(1) + expect(verify).not.toHaveBeenCalled() + expect(f.tracker.isValidRootForHeight).toHaveBeenCalledWith(txid, blockHeight) + expect(f.submit).toHaveBeenCalledTimes(1) + expect(f.tracker.currentHeight).not.toHaveBeenCalled() + }) + it('reports a finite proof request limit for a block above 1000 admissions (B02 chunking required)', async () => { const f = fixture() const admitted = Array.from({ length: 1001 }, (_, blockIndex) => ({ From d836611b71fdfbf2dbd521eacc3eb0685109cac8 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 17:17:18 -0500 Subject: [PATCH 013/168] feat(overlay): add opt-in Mongo admission commit and Engine submit path Wire Engine.submit to commitAdmission when overlay-admission-v1 is advertised. MongoAdmissionStorage honors majority ACK, conditional spends, enlisted indexes or a durable lookup/propagation outbox. Knex/SQL remains the default. --- docs/packages/overlays/overlay.md | 20 +- docs/reference/package-api-migrations.md | 6 +- governance/package-release-notes.json | 4 +- packages/overlays/overlay/CHANGELOG.md | 7 +- packages/overlays/overlay/README.md | 21 +- packages/overlays/overlay/src/Engine.ts | 41 + .../overlays/overlay/src/EngineAdmission.ts | 315 ++++++ .../src/__tests/Engine.admission.test.ts | 205 ++++ .../__tests/mongo/MongoAdmissionHarness.ts | 479 +++++++++ .../mongo/MongoAdmissionStorage.test.ts | 239 +++++ .../overlays/overlay/src/storage/mongo.ts | 3 + .../src/storage/mongo/MongoAdmissionPlan.ts | 274 +++++ .../storage/mongo/MongoAdmissionStorage.ts | 936 ++++++++++++++++++ .../src/storage/mongo/MongoOverlayStorage.ts | 351 +++++++ specs/overlay/mongo-v1.md | 53 +- specs/overlay/persistence-v1.md | 3 +- 16 files changed, 2906 insertions(+), 51 deletions(-) create mode 100644 packages/overlays/overlay/src/EngineAdmission.ts create mode 100644 packages/overlays/overlay/src/__tests/Engine.admission.test.ts create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts create mode 100644 packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts create mode 100644 packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts create mode 100644 packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts diff --git a/docs/packages/overlays/overlay.md b/docs/packages/overlays/overlay.md index 38a99b04a..f0720fec0 100644 --- a/docs/packages/overlays/overlay.md +++ b/docs/packages/overlays/overlay.md @@ -80,17 +80,19 @@ const result = await engine.lookup({ The package exports an additive `AdmissionStorage` capability, semantic identity helpers and recovery fence/cursor predicates. These define the local durable -receipt and pending index/propagation boundary for future adapters. Current -`Engine.submit`, its early STEAK callback and Knex storage do not use the -capability. See the [persistence v1 specification](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/persistence-v1.md) +receipt and pending index/propagation boundary. When storage advertises a +complete `overlay-admission-v1` provider, `Engine.submit` commits through that +capability and returns the saved STEAK after majority ack. Knex storage and +its early STEAK callback do not use the capability. See the [persistence v1 +specification](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/persistence-v1.md) for the shared fixtures and explicit limits. No consumer migration is required. ## Optional Mongo foundation The package also contains an opt-in MongoDB foundation for schema bootstrap, -content-addressed payload publication, reference guards, and payload collection. -It is not an Engine integration, an `AdmissionStorage` implementation, or a -default storage selection; importing `@bsv/overlay` does not load MongoDB. +content-addressed payload publication, reference guards, payload collection, +and an explicit Engine admission adapter. Mongo is not the default storage +selection; importing `@bsv/overlay` does not load MongoDB. Applications using a Mongo deep entry point install the optional peer first: @@ -98,8 +100,10 @@ Applications using a Mongo deep entry point install the optional peer first: npm install @bsv/overlay mongodb@^7.5.0 ``` -The initial entry points are `@bsv/overlay/storage/mongo/MongoSchema` and -`@bsv/overlay/storage/mongo/MongoPayloadStore`. They require an explicitly +The initial entry points are `@bsv/overlay/storage/mongo/MongoSchema`, +`@bsv/overlay/storage/mongo/MongoPayloadStore`, +`@bsv/overlay/storage/mongo/MongoAdmissionStorage`, and +`@bsv/overlay/storage/mongo/MongoOverlayStorage`. They require an explicitly operated unsharded replica set; the supported deployment profile is three members. See the [Mongo v1 foundation](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/mongo-v1.md). diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index b861c5c49..d369601ac 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -42,7 +42,7 @@ and clean-consumer tests remain the executable type authority. | `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | | `@bsv/message-box-client` | `2.4.0` | `2.5.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. | -| `@bsv/overlay` | `2.2.1` | `2.4.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. | +| `@bsv/overlay` | `2.2.1` | `2.4.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Knex/SQL submit remains the default path; Engine uses commitAdmission only when storage advertises a complete overlay-admission-v1 provider. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No default-adapter change, SQL-to-Mongo migration, mixed-writer database topology, or production activation is included. | | `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.2` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | | `@bsv/overlay-express` | `2.5.0` | `2.6.2` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | | `@bsv/overlay-topics` | `1.6.10` | `1.7.2` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | @@ -277,8 +277,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay.md](../packages/overlays/overlay.md) - Source: [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) -- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. -- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. +- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. Adds an opt-in Mongo AdmissionStorage adapter and Engine submit path that uses commitAdmission when overlay-admission-v1 is advertised, with enlisted indexes or a durable projection outbox and majority ACK. +- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Knex/SQL submit remains the default path; Engine uses commitAdmission only when storage advertises a complete overlay-admission-v1 provider. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No default-adapter change, SQL-to-Mongo migration, mixed-writer database topology, or production activation is included. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index 2720237f9..dc5c0b4cc 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -126,8 +126,8 @@ "name": "@bsv/overlay", "publishedVersion": "2.2.1", "releaseType": "minor", - "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection.", - "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included." + "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. Adds an opt-in Mongo AdmissionStorage adapter and Engine submit path that uses commitAdmission when overlay-admission-v1 is advertised, with enlisted indexes or a durable projection outbox and majority ACK.", + "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Knex/SQL submit remains the default path; Engine uses commitAdmission only when storage advertises a complete overlay-admission-v1 provider. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No default-adapter change, SQL-to-Mongo migration, mixed-writer database topology, or production activation is included." }, { "name": "@bsv/overlay-discovery-services", diff --git a/packages/overlays/overlay/CHANGELOG.md b/packages/overlays/overlay/CHANGELOG.md index 64c003ec8..3fbc39c25 100644 --- a/packages/overlays/overlay/CHANGELOG.md +++ b/packages/overlays/overlay/CHANGELOG.md @@ -23,8 +23,11 @@ All notable changes to this project will be documented in this file. The format - Adds optional atomic admission/receipt and recovery contracts with portable identity, exact integer and fencing fixtures. Existing Engine/Knex paths are unchanged; no migration is required. - Adds an opt-in MongoDB schema, payload-publication, reference-guard, and payload-GC foundation behind an optional `mongodb` peer. It does not activate - MongoDB in Engine, select a new default adapter, or provide an admission - receipt path. + MongoDB as the overlay default or select a new SQL adapter. +- Adds an opt-in Mongo `AdmissionStorage` adapter and Engine submit path that + uses `commitAdmission` when `overlay-admission-v1` is advertised, with + enlisted same-session indexes or a durable projection outbox, majority ACK, + and SQL/Knex compatibility retained. ### Added - (Include new features or significant user-visible enhancements here.) diff --git a/packages/overlays/overlay/README.md b/packages/overlays/overlay/README.md index 891b16f70..39d059ffe 100644 --- a/packages/overlays/overlay/README.md +++ b/packages/overlays/overlay/README.md @@ -85,8 +85,10 @@ applications should prefer the root entry point wherever possible. adapters. `getAdmissionStorage(storage)` detects an explicit provider with both commit and reconciliation methods. Existing Knex and injected legacy adapters remain supported; their individual methods do not imply atomic submission. -Current `Engine.submit` does not call this capability, and its early STEAK -callback is not a durable commit receipt. +When `getAdmissionStorage(storage)` observes a complete `overlay-admission-v1` +provider, `Engine.submit` builds an admission plan and returns the saved STEAK +only after majority commit. The SQL/Knex path and its early STEAK callback are +unchanged; that callback is not a durable commit receipt. The contract separates local commit, index visibility and propagation. It binds operation identity to verified transaction, topic/policy and off-chain context; @@ -100,9 +102,9 @@ in this release candidate. ## Optional Mongo foundation The package also contains an opt-in MongoDB foundation for schema bootstrap, -content-addressed payload publication, reference guards, and payload collection. -It is not an Engine integration, an `AdmissionStorage` implementation, or a -default storage selection; importing `@bsv/overlay` alone does not load MongoDB. +content-addressed payload publication, reference guards, payload collection, +and an explicit `AdmissionStorage` adapter. Mongo is not the default Engine +storage selection; importing `@bsv/overlay` alone does not load MongoDB. Applications using a Mongo deep entry point install the optional peer first: @@ -111,15 +113,16 @@ npm install @bsv/overlay mongodb@^7.5.0 ``` The initial entry points are -`@bsv/overlay/storage/mongo/MongoSchema` and -`@bsv/overlay/storage/mongo/MongoPayloadStore`. They require an explicitly +`@bsv/overlay/storage/mongo/MongoSchema`, +`@bsv/overlay/storage/mongo/MongoPayloadStore`, +`@bsv/overlay/storage/mongo/MongoAdmissionStorage`, and +`@bsv/overlay/storage/mongo/MongoOverlayStorage`. They require an explicitly operated unsharded replica set; the supported deployment profile is three members. Payload publication makes GridFS bytes physically `published` before the guarded payload row becomes `ready`; caller-session reference and GC operations share that row guard. See the [Mongo v1 foundation](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/mongo-v1.md) -for operational bounds, recovery rules, and the still-pending admission -transaction integration. +for operational bounds, recovery rules, and the opt-in admission path. ## Runtime and package formats diff --git a/packages/overlays/overlay/src/Engine.ts b/packages/overlays/overlay/src/Engine.ts index 5022b010d..492db54a6 100644 --- a/packages/overlays/overlay/src/Engine.ts +++ b/packages/overlays/overlay/src/Engine.ts @@ -42,6 +42,12 @@ import { } from './BASM.js' import { BASMRemote } from './BASMRemote.js' import { serializeErrorForLog, serializeLogValue } from './SafeLog.js' +import { + buildOverlayAdmissionPlan, + getOverlayAdmissionHost, + overlayAdmissionMode, + waitForAdmissionReceipt +} from './EngineAdmission.js' const DEFAULT_GASP_SYNC_LIMIT = 10000 const DEFAULT_BASM_RANGE_LIMIT = 1024 @@ -1116,6 +1122,41 @@ export class Engine { throw error } + const admissionHost = getOverlayAdmissionHost(this.storage) + if (admissionHost !== undefined) { + if (!anyTopicAccepted) { + if (onSteakReady !== undefined) onSteakReady(steak) + return steak + } + const admissionMode = overlayAdmissionMode(mode) + const includePropagation = + this.advertiser !== undefined && + mode !== 'historical-tx' && + mode !== 'historical-tx-no-spv' + const buildPlan = async () => + await buildOverlayAdmissionPlan({ + host: admissionHost, + tx, + txid, + beef: taggedBEEF.beef, + topics: taggedBEEF.topics, + mode: admissionMode, + offChainValues, + validations, + failedTopics, + lookupServices: this.lookupServices, + includePropagation + }) + const committed = await waitForAdmissionReceipt( + admissionHost.admission, + await buildPlan(), + buildPlan + ) + const acknowledged = JSON.parse(committed.receipt.steak) as STEAK + if (onSteakReady !== undefined) onSteakReady(acknowledged) + return acknowledged + } + // Call the callback function with STEAK if it is provided (before storage mutations) if (onSteakReady !== undefined) { onSteakReady(steak) diff --git a/packages/overlays/overlay/src/EngineAdmission.ts b/packages/overlays/overlay/src/EngineAdmission.ts new file mode 100644 index 000000000..4832bc619 --- /dev/null +++ b/packages/overlays/overlay/src/EngineAdmission.ts @@ -0,0 +1,315 @@ +import { createHash } from 'node:crypto' +import type { STEAK, Transaction } from '@bsv/sdk' +import { + admissionSemanticDigest, + getAdmissionStorage, + type AdmissionCommit, + type AdmissionCommitResult, + type AdmissionOutboxIntent, + type AdmissionPayloadRef, + type AdmissionReconcileResult, + type AdmissionStorage, + type AdmissionTopicDecision, + type HistoryFence, + type StorageScope +} from './storage/AdmissionStorage.js' +import type { Output } from './Output.js' +import type { LookupService } from './LookupService.js' + +export const OVERLAY_ENGINE_POLICY_ID = 'overlay-engine-submit-v1' + +export type OverlayAdmissionMode = 'live' | 'historical' + +type TopicValidationLike = { + topic: string + isDupe: boolean + previousCoins: number[] + previousOutputs: Array + admissibleOutputs: { + outputsToAdmit: number[] + coinsToRetain: number[] + coinsRemoved?: number[] + } +} + +export interface OverlayAdmissionHost { + admission: AdmissionStorage + admissionScope: StorageScope + publishAdmissionPayload?: (input: { + kind: AdmissionPayloadRef['kind'] + bytes: Uint8Array + txid?: string + }) => Promise + enlistedIndexTargets?: () => readonly string[] + getHistoryFence?: (topic: string) => Promise +} + +export function getOverlayAdmissionHost(storage: unknown): OverlayAdmissionHost | undefined { + const admission = getAdmissionStorage(storage) + if (admission === undefined || typeof storage !== 'object' || storage === null) return undefined + if (!('admissionScope' in storage)) return undefined + const scope = (storage as { admissionScope?: StorageScope }).admissionScope + if ( + scope === undefined || + typeof scope.network !== 'string' || + typeof scope.genesisHash !== 'string' || + typeof scope.nodeId !== 'string' || + !/^[0-9a-f]{64}$/.test(scope.genesisHash) + ) { + return undefined + } + const host = storage as OverlayAdmissionHost + return { + admission, + admissionScope: { ...scope }, + publishAdmissionPayload: host.publishAdmissionPayload, + enlistedIndexTargets: host.enlistedIndexTargets, + getHistoryFence: host.getHistoryFence + } +} + +export function overlayAdmissionContextDigest(offChainValues?: number[]): string { + const hash = createHash('sha256') + if (offChainValues !== undefined) hash.update(Buffer.from(offChainValues)) + return hash.digest('hex') +} + +export function overlayAdmissionOperationId( + mode: OverlayAdmissionMode, + txid: string, + topics: string[] +): string { + const raw = `submit:${mode}:${txid}:${[...topics].sort().join('\n')}` + if (Buffer.byteLength(raw, 'utf8') <= 512 && raw.isWellFormed()) return raw + return `submit:${createHash('sha256').update(raw, 'utf8').digest('hex')}` +} + +export function overlayAdmissionMode( + mode: 'historical-tx' | 'current-tx' | 'historical-tx-no-spv' +): OverlayAdmissionMode { + return mode === 'current-tx' ? 'live' : 'historical' +} + +async function localPayload( + host: OverlayAdmissionHost, + kind: AdmissionPayloadRef['kind'], + bytes: Uint8Array, + txid?: string +): Promise { + if (host.publishAdmissionPayload !== undefined) { + return await host.publishAdmissionPayload({ kind, bytes, txid }) + } + return { + kind, + digest: createHash('sha256').update(bytes).digest('hex'), + byteLength: String(bytes.byteLength) + } +} + +export async function buildOverlayAdmissionPlan(input: { + host: OverlayAdmissionHost + tx: Transaction + txid: string + beef: number[] + topics: string[] + mode: OverlayAdmissionMode + offChainValues?: number[] + validations: TopicValidationLike[] + failedTopics: Set + lookupServices: { [key: string]: LookupService } + includePropagation: boolean +}): Promise { + const accepted = input.validations.filter( + validation => + !input.failedTopics.has(validation.topic) && + (validation.isDupe || + validation.admissibleOutputs.outputsToAdmit.length > 0 || + validation.admissibleOutputs.coinsToRetain.length > 0 || + validation.previousCoins.length > 0) + ) + const identityTopics = accepted.map(validation => ({ + topic: validation.topic, + policyId: OVERLAY_ENGINE_POLICY_ID + })) + if (identityTopics.length === 0) throw new Error('Overlay admission plan has no topics') + const identity = { + scope: input.host.admissionScope, + txid: input.txid, + mode: input.mode, + contextDigest: overlayAdmissionContextDigest(input.offChainValues), + topics: identityTopics + } + const raw = await localPayload( + input.host, + 'raw-transaction', + Buffer.from(input.tx.toBinary()), + input.txid + ) + const payloads: AdmissionPayloadRef[] = [raw] + if (input.tx.merklePath !== undefined) { + payloads.push( + await localPayload(input.host, 'merkle-path', Buffer.from(input.tx.merklePath.toBinary())) + ) + } + const decisions: AdmissionTopicDecision[] = [] + const steak: STEAK = {} + for (const validation of accepted) { + const { outputsConsumed, outputsToMarkStale } = classifyCoins(input.tx, validation) + const fence = + input.host.getHistoryFence === undefined + ? { chainEpoch: '0', topicHistoryGeneration: '0' } + : await input.host.getHistoryFence(validation.topic) + const outputs: AdmissionTopicDecision['outputs'] = [] + for (const outputIndex of validation.admissibleOutputs.outputsToAdmit) { + const txOut = input.tx.outputs[outputIndex] + if (typeof txOut?.satoshis !== 'number' || !Number.isSafeInteger(txOut.satoshis)) continue + const scriptBytes = Buffer.from(txOut.lockingScript.toBinary()) + const script = await localPayload(input.host, 'locking-script', scriptBytes) + payloads.push(script) + outputs.push({ + txid: input.txid, + outputIndex: String(outputIndex), + satoshis: String(txOut.satoshis), + score: String(Date.now()), + script: { payload: script, offset: '0', byteLength: script.byteLength } + }) + } + const spends = validation.previousOutputs.flatMap(output => + output === null + ? [] + : [ + { + outpoint: { txid: output.txid, outputIndex: String(output.outputIndex) }, + expectedVersion: '1', + spender: input.txid + } + ] + ) + decisions.push({ + topic: validation.topic, + expectedHistory: fence, + reads: [], + spends, + evictions: outputsToMarkStale.map(item => ({ + txid: item.txid, + outputIndex: String(item.previousOutputIndex) + })), + outputs, + edges: outputsConsumed.flatMap(source => + outputs.map(output => ({ + source: { txid: source.txid, outputIndex: String(source.outputIndex) }, + consumer: { txid: output.txid, outputIndex: output.outputIndex } + })) + ), + applied: { txid: input.txid } + }) + steak[validation.topic] = { + outputsToAdmit: validation.admissibleOutputs.outputsToAdmit, + coinsToRetain: validation.admissibleOutputs.coinsToRetain, + coinsRemoved: outputsToMarkStale.map(item => item.inputIndex) + } + } + const enlisted = new Set(input.host.enlistedIndexTargets?.() ?? []) + const outbox: AdmissionOutboxIntent[] = Object.keys(input.lookupServices) + .filter(target => !enlisted.has(target)) + .map(target => ({ + eventId: `${overlayAdmissionOperationId(input.mode, input.txid, input.topics)}:lookup:${target}`, + kind: 'lookup' as const, + target, + payloads: [raw] + })) + if (input.includePropagation && input.mode === 'live') { + outbox.push({ + eventId: `${overlayAdmissionOperationId(input.mode, input.txid, input.topics)}:propagate`, + kind: 'propagation', + target: 'ship', + payloads: [raw] + }) + } + return { + key: { + scope: input.host.admissionScope, + operationId: overlayAdmissionOperationId(input.mode, input.txid, input.topics), + semanticDigest: admissionSemanticDigest(identity) + }, + identity, + payloads: uniquePayloads(payloads), + decisions, + outbox, + steak: JSON.stringify(steak) + } +} + +export async function waitForAdmissionReceipt( + admission: AdmissionStorage, + plan: AdmissionCommit, + rebuild: () => Promise +): Promise { + let current = plan + for (let attempt = 0; attempt < 8; attempt += 1) { + const result = await admission.commitAdmission(current) + if (result.state === 'committed') return result + if (result.state === 'rejected') { + if (result.code === 'read-conflict' && attempt < 7) { + current = await rebuild() + continue + } + throw new Error(`Overlay admission rejected: ${result.code}`) + } + const reconciled: AdmissionReconcileResult = await admission.reconcileAdmission( + current.key, + result.attemptId + ) + if (reconciled.state === 'committed') return reconciled + if (reconciled.state === 'aborted') { + current = await rebuild() + continue + } + if (reconciled.state === 'rejected') { + throw new Error(`Overlay admission rejected: ${reconciled.code}`) + } + } + throw new Error('Overlay admission commit is pending') +} + +function uniquePayloads(payloads: AdmissionPayloadRef[]): AdmissionPayloadRef[] { + const seen = new Set() + const result: AdmissionPayloadRef[] = [] + for (const payload of payloads) { + const key = `${payload.kind}:${payload.digest}:${payload.byteLength}` + if (seen.has(key)) continue + seen.add(key) + result.push(payload) + } + return result +} + +function classifyCoins( + tx: Transaction, + validation: TopicValidationLike +): { + outputsConsumed: Array<{ txid: string; outputIndex: number }> + outputsToMarkStale: Array<{ txid: string; previousOutputIndex: number; inputIndex: number }> +} { + const outputsConsumed: Array<{ txid: string; outputIndex: number }> = [] + const outputsToMarkStale: Array<{ + txid: string + previousOutputIndex: number + inputIndex: number + }> = [] + for (const inputIndex of validation.previousCoins) { + const input = tx.inputs[inputIndex] + const previousTXID = input.sourceTXID ?? input.sourceTransaction?.id('hex') + if (typeof previousTXID !== 'string') continue + if (validation.admissibleOutputs.coinsToRetain.includes(inputIndex)) { + outputsConsumed.push({ txid: previousTXID, outputIndex: input.sourceOutputIndex }) + } else { + outputsToMarkStale.push({ + txid: previousTXID, + previousOutputIndex: input.sourceOutputIndex, + inputIndex + }) + } + } + return { outputsConsumed, outputsToMarkStale } +} diff --git a/packages/overlays/overlay/src/__tests/Engine.admission.test.ts b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts new file mode 100644 index 000000000..6392d62dc --- /dev/null +++ b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts @@ -0,0 +1,205 @@ +import { Engine } from '../Engine.js' +import type { LookupService } from '../LookupService.js' +import type { TopicManager } from '../TopicManager.js' +import type { Storage } from '../storage/Storage.js' +import type { + AdmissionCommit, + AdmissionReceipt, + AdmissionStorage, + StorageScope +} from '../storage/AdmissionStorage.js' +import { getAdmissionStorage } from '../storage/AdmissionStorage.js' +import { KnexStorage } from '../storage/knex/KnexStorage.js' +import type { Knex } from 'knex' +import { Transaction, type AdmittanceInstructions } from '@bsv/sdk' + +const mockChainTracker = { + isValidRootForHeight: jest.fn(async () => true), + currentHeight: jest.fn(async () => 800000) +} + +const BRC62Hex = + '0100beef01fe636d0c0007021400fe507c0c7aa754cef1f7889d5fd395cf1f785dd7de98eed895dbedfe4e5bc70d1502ac4e164f5bc16746bb0868404292ac8318bbac3800e4aad13a014da427adce3e010b00bc4ff395efd11719b277694cface5aa50d085a0bb81f613f70313acd28cf4557010400574b2d9142b8d28b61d88e3b2c3f44d858411356b49a28a4643b6d1a6a092a5201030051a05fc84d531b5d250c23f4f886f6812f9fe3f402d61607f977b4ecd2701c19010000fd781529d58fc2523cf396a7f25440b409857e7e221766c57214b1d38c7b481f01010062f542f45ea3660f86c013ced80534cb5fd4c19d66c56e7e8c5d4bf2d40acc5e010100b121e91836fd7cd5102b654e9f72f3cf6fdbfd0b161c53a9c54b12c841126331020100000001cd4e4cac3c7b56920d1e7655e7e260d31f29d9a388d04910f1bbd72304a79029010000006b483045022100e75279a205a547c445719420aa3138bf14743e3f42618e5f86a19bde14bb95f7022064777d34776b05d816daf1699493fcdf2ef5a5ab1ad710d9c97bfb5b8f7cef3641210263e2dee22b1ddc5e11f6fab8bcd2378bdd19580d640501ea956ec0e786f93e76ffffffff013e660000000000001976a9146bfd5c7fbe21529d45803dbcf0c87dd3c71efbc288ac0000000001000100000001ac4e164f5bc16746bb0868404292ac8318bbac3800e4aad13a014da427adce3e000000006a47304402203a61a2e931612b4bda08d541cfb980885173b8dcf64a3471238ae7abcd368d6402204cbf24f04b9aa2256d8901f0ed97866603d2be8324c2bfb7a37bf8fc90edd5b441210263e2dee22b1ddc5e11f6fab8bcd2378bdd19580d640501ea956ec0e786f93e76ffffffff013c660000000000001976a9146bfd5c7fbe21529d45803dbcf0c87dd3c71efbc288ac0000000000' + +const exampleTX = Transaction.fromHexBEEF(BRC62Hex) +const exampleBeef = exampleTX.toBEEF() + +const admissionScope: StorageScope = { + network: 'testnet', + genesisHash: '11'.repeat(32), + nodeId: 'engine-node' +} + +function receiptFor(plan: AdmissionCommit): AdmissionReceipt { + return { + operationId: plan.key.operationId, + semanticDigest: plan.key.semanticDigest, + durability: 'atomic-local', + steak: plan.steak, + indexes: plan.outbox + .filter(intent => intent.kind === 'lookup') + .map(intent => ({ target: intent.target, state: 'pending' as const })), + propagation: plan.outbox.some(intent => intent.kind === 'propagation') + ? 'pending' + : 'not-requested' + } +} + +describe('Engine admission submit', () => { + let mockTopicManager: TopicManager + let mockLookupService: LookupService + let mockStorage: Storage & { admission: AdmissionStorage; admissionScope: StorageScope } + let commitAdmission: jest.Mock + let saved: AdmissionReceipt | undefined + + beforeEach(() => { + jest.clearAllMocks() + saved = undefined + mockTopicManager = { + identifyAdmissibleOutputs: jest.fn(async (): Promise => ({ + outputsToAdmit: [0], + coinsToRetain: [] + })), + getDocumentation: async () => 'docs', + getMetaData: async () => ({ name: 'Mock', shortDescription: 'Mock' }) + } + mockLookupService = { + outputAdmittedByTopic: jest.fn(), + outputSpent: jest.fn(), + lookup: jest.fn(), + outputNoLongerRetainedInHistory: jest.fn(), + outputEvicted: jest.fn(), + admissionMode: 'locking-script', + spendNotificationMode: 'none', + getDocumentation: async () => 'docs', + getMetaData: async () => ({ name: 'Mock', shortDescription: 'Mock' }) + } + commitAdmission = jest.fn(async (plan: AdmissionCommit) => { + if (saved !== undefined) return { state: 'committed' as const, receipt: saved } + saved = receiptFor(plan) + return { state: 'committed' as const, receipt: saved } + }) + mockStorage = { + admission: { + protocol: 'overlay-admission-v1', + commitAdmission, + reconcileAdmission: jest.fn() + }, + admissionScope, + doesAppliedTransactionExist: jest.fn(async () => false), + insertAppliedTransaction: jest.fn(), + insertOutput: jest.fn(), + findOutput: jest.fn(async () => null), + findOutputsForTransaction: jest.fn(async () => []), + markUTXOAsSpent: jest.fn(), + updateConsumedBy: jest.fn(), + updateTransactionBEEF: jest.fn(), + deleteOutput: jest.fn(), + findUTXOsForTopic: jest.fn(), + updateLastInteraction: jest.fn(), + getLastInteraction: jest.fn(async () => 0) + } + }) + + test('SQL Knex storage still has no admission capability', () => { + const knex = jest.fn() as unknown as Knex + expect(getAdmissionStorage(new KnexStorage(knex))).toBeUndefined() + }) + + test('uses commitAdmission and does not mutate via CRUD or lookup callbacks', async () => { + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker + ) + const steak = await engine.submit({ beef: exampleBeef, topics: ['Hello'] }) + expect(commitAdmission).toHaveBeenCalledTimes(1) + expect(mockStorage.insertOutput).not.toHaveBeenCalled() + expect(mockStorage.markUTXOAsSpent).not.toHaveBeenCalled() + expect(mockLookupService.outputAdmittedByTopic).not.toHaveBeenCalled() + expect(steak.Hello.outputsToAdmit).toEqual([0]) + }) + + test('returns the saved STEAK after a lost admission ACK', async () => { + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker + ) + const first = await engine.submit({ beef: exampleBeef, topics: ['Hello'] }) + const second = await engine.submit({ beef: exampleBeef, topics: ['Hello'] }) + expect(second).toEqual(first) + expect(commitAdmission).toHaveBeenCalledTimes(2) + expect(mockLookupService.outputAdmittedByTopic).not.toHaveBeenCalled() + }) + + test('historical submit omits propagation outbox', async () => { + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker, + undefined, + undefined, + undefined, + undefined, + { + createAdvertisements: jest.fn(), + revokeAdvertisements: jest.fn(), + findAllAdvertisements: jest.fn(), + parseAdvertisement: jest.fn() + } + ) + await engine.submit({ beef: exampleBeef, topics: ['Hello'] }, undefined, 'historical-tx') + const plan = commitAdmission.mock.calls[0][0] as AdmissionCommit + expect(plan.identity.mode).toBe('historical') + expect(plan.outbox.some(intent => intent.kind === 'propagation')).toBe(false) + }) + + test('live submit with an advertiser records a pending propagation outbox', async () => { + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker, + undefined, + undefined, + undefined, + undefined, + { + createAdvertisements: jest.fn(), + revokeAdvertisements: jest.fn(), + findAllAdvertisements: jest.fn(), + parseAdvertisement: jest.fn() + } + ) + const onReady = jest.fn() + const steak = await engine.submit({ beef: exampleBeef, topics: ['Hello'] }, onReady) + const plan = commitAdmission.mock.calls[0][0] as AdmissionCommit + expect(plan.identity.mode).toBe('live') + expect(plan.outbox.some(intent => intent.kind === 'propagation')).toBe(true) + expect(onReady).toHaveBeenCalledWith(steak) + }) + + test('calls onSTEAKReady only after the admission receipt', async () => { + const order: string[] = [] + commitAdmission = jest.fn(async (plan: AdmissionCommit) => { + order.push('commit') + saved = receiptFor(plan) + return { state: 'committed' as const, receipt: saved } + }) + mockStorage.admission.commitAdmission = commitAdmission + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker + ) + await engine.submit({ beef: exampleBeef, topics: ['Hello'] }, () => { + order.push('callback') + }) + expect(order).toEqual(['commit', 'callback']) + }) +}) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts new file mode 100644 index 000000000..db5f233dc --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts @@ -0,0 +1,479 @@ +import { randomUUID } from 'node:crypto' +import { Binary } from 'mongodb' +import { + type AdmissionCommitResult, + type AdmissionOperationKey, + type AdmissionOutpoint, + type AdmissionPayloadRef, + type AdmissionReconcileResult, + type AdmissionStorage, + type StorageScope +} from '../../storage/AdmissionStorage.js' +import type { RecoveryLease } from '../../storage/RecoveryContract.js' +import type { + AdmissionStorageContractHarness, + AdmissionStorageFaults, + AdmissionStorageSeeds, + AdmissionStorageTestSnapshot +} from '../admission/ReferenceAdmissionStorage.js' +import { MongoAdmissionStorage } from '../../storage/mongo/MongoAdmissionStorage.js' +import { MongoReadGuards } from '../../storage/mongo/MongoReadGuards.js' +import { + MongoCollectionNames, + decodeMongoUint64, + encodeMongoOutputIndex, + encodeMongoUint64, + mongoChainKey, + mongoNodeKey, + mongoRecordKey +} from '../../storage/mongo/MongoSchema.js' +import type { MongoReplicaFixture } from './MongoReplicaFixture.js' +import { referenceScope } from '../admission/ReferenceAdmissionStorage.js' + +const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T +const scopeKey = (scope: StorageScope): string => + JSON.stringify([scope.network, scope.genesisHash, scope.nodeId]) +const operationKey = (key: AdmissionOperationKey): string => + JSON.stringify([scopeKey(key.scope), key.operationId]) +const outputKey = (scope: StorageScope, topic: string, outpoint: AdmissionOutpoint): string => + JSON.stringify([scopeKey(scope), topic, outpoint.txid, outpoint.outputIndex]) +const appliedKey = (scope: StorageScope, topic: string, txid: string): string => + JSON.stringify([scope.network, scope.genesisHash, scope.nodeId, topic, txid]) +const edgeKey = ( + scope: StorageScope, + topic: string, + source: AdmissionOutpoint, + consumer: AdmissionOutpoint +): string => + JSON.stringify([scope.network, scope.genesisHash, scope.nodeId, topic, source, consumer]) + +class FaultAdmission implements AdmissionStorage { + readonly protocol = 'overlay-admission-v1' as const + inner: AdmissionStorage + loseReply = false + abortBefore = false + unresolved = new Map() + + constructor(inner: AdmissionStorage) { + this.inner = inner + } + + async commitAdmission( + plan: Parameters[0] + ): Promise { + const key = operationKey(plan.key) + const active = this.unresolved.get(key) + if (active !== undefined) return { state: 'pending', attemptId: active.attemptId } + if (this.abortBefore) { + this.abortBefore = false + const attemptId = randomUUID() + this.unresolved.set(key, { attemptId, aborted: true }) + return { state: 'pending', attemptId } + } + const result = await this.inner.commitAdmission(plan) + if (this.loseReply && result.state === 'committed') { + this.loseReply = false + const attemptId = randomUUID() + this.unresolved.set(key, { attemptId, aborted: false }) + return { state: 'pending', attemptId } + } + return result + } + + async reconcileAdmission( + key: AdmissionOperationKey, + attemptId?: string + ): Promise { + const record = this.unresolved.get(operationKey(key)) + if (record !== undefined) { + if (attemptId !== undefined && attemptId !== record.attemptId) { + return { state: 'pending', attemptId: record.attemptId } + } + this.unresolved.delete(operationKey(key)) + if (record.aborted) return { state: 'aborted' } + return await this.inner.reconcileAdmission(key, attemptId) + } + return await this.inner.reconcileAdmission(key, attemptId) + } +} + +export class MongoAdmissionHarness implements AdmissionStorageContractHarness { + private inner: MongoAdmissionStorage + private readonly wrapper: FaultAdmission + private readonly leases = new Map() + private readonly seededPayloads = new Set() + readonly seed: AdmissionStorageSeeds + readonly faults: AdmissionStorageFaults + + constructor(private readonly fixture: MongoReplicaFixture) { + this.inner = new MongoAdmissionStorage(fixture.db, referenceScope) + this.wrapper = new FaultAdmission(this.inner) + this.seed = { + readyPayload: async ref => { + this.seededPayloads.add(`${ref.kind}:${ref.digest}`) + await this.insertReadyPayload(ref) + }, + read: async (scope, _topic, key, version) => { + const guards = new MongoReadGuards(this.fixture.db) + await guards.initialize(scope, key) + await this.fixture.db + .collection(MongoCollectionNames.readGuards) + .updateOne( + { _id: mongoRecordKey(mongoNodeKey(scope), 'read-guard', key) }, + { $set: { version, updatedAt: new Date() } } + ) + }, + spendable: async (scope, topic, outpoint, version) => { + await this.insertSpendable(scope, topic, outpoint, version) + }, + history: async (scope, topic, fence) => { + await this.insertHistory(scope, topic, fence) + }, + lease: async lease => { + this.leases.set(`${lease.topic}:${lease.peerId}:${lease.jobId}`, clone(lease)) + await this.insertLease(lease) + }, + now: () => undefined + } + this.faults = { + loseReplyAfterCommitOnce: () => { + this.wrapper.loseReply = true + }, + abortBeforeCommitOnce: () => { + this.wrapper.abortBefore = true + } + } + } + + get storage(): { admission: AdmissionStorage } { + return { admission: this.wrapper } + } + + restart(): AdmissionStorage { + this.inner = new MongoAdmissionStorage(this.fixture.db, referenceScope) + this.wrapper.inner = this.inner + return this.wrapper + } + + async reset(): Promise { + this.wrapper.loseReply = false + this.wrapper.abortBefore = false + this.wrapper.unresolved.clear() + this.leases.clear() + this.seededPayloads.clear() + await this.inner.close().catch(() => undefined) + for (const name of Object.values(MongoCollectionNames)) { + if (name === MongoCollectionNames.schema) continue + await this.fixture.db.collection(name).deleteMany({}) + } + await this.fixture.db.collection('enlisted_index').deleteMany({}) + this.inner = new MongoAdmissionStorage(this.fixture.db, referenceScope) + this.wrapper.inner = this.inner + } + + async snapshot(): Promise { + const payloads = await this.fixture.db + .collection(MongoCollectionNames.payloads) + .find() + .toArray() + const refs = await this.fixture.db + .collection(MongoCollectionNames.payloadReferences) + .find() + .toArray() + const pinned = new Set(refs.map(item => item.payloadId as string)) + const reads = await this.fixture.db.collection(MongoCollectionNames.readGuards).find().toArray() + const fences = await this.fixture.db + .collection(MongoCollectionNames.topicGenerations) + .find() + .toArray() + const jobs = await this.fixture.db + .collection(MongoCollectionNames.basmRecoveryJobs) + .find() + .toArray() + const outputs = await this.fixture.db.collection(MongoCollectionNames.outputs).find().toArray() + const edges = await this.fixture.db + .collection(MongoCollectionNames.consumptionEdges) + .find() + .toArray() + const applied = await this.fixture.db + .collection(MongoCollectionNames.appliedTransactions) + .find() + .toArray() + const lookup = await this.fixture.db + .collection(MongoCollectionNames.lookupOutbox) + .find() + .toArray() + const propagation = await this.fixture.db + .collection(MongoCollectionNames.propagationOutbox) + .find() + .toArray() + const historyRefs = refs.filter(item => item.slot === 'history-update') + const historyUpdates = [] + for (const reference of historyRefs) { + const payload = payloads.find(item => item._id === reference.payloadId) + const bytes = + payload?.inlineData instanceof Binary + ? Buffer.from(payload.inlineData.buffer) + : Buffer.alloc(0) + const parsed = JSON.parse(bytes.toString('utf8')) as { + affectedFromHeight: string + checkpoint?: string + } + historyUpdates.push({ topic: reference.ownerId as string, ...parsed }) + } + return { + payloads: payloads + .filter( + item => + this.seededPayloads.has(`${item.kind}:${item.digest}`) || pinned.has(item._id as string) + ) + .map(item => ({ + ref: { + kind: item.kind, + digest: item.digest, + byteLength: decodeMongoUint64(item.byteLength as string) + } as AdmissionPayloadRef, + pinned: pinned.has(item._id as string) + })), + reads: reads + .filter(item => item.version !== null) + .map(item => ({ + key: JSON.stringify([scopeKey(this.scopeOf(item)), '', item.key]), + version: item.version as string + })), + fences: fences.map(item => ({ + topic: item.topic as string, + fence: { + chainEpoch: decodeMongoUint64(item.chainEpoch as string), + topicHistoryGeneration: decodeMongoUint64(item.topicHistoryGeneration as string) + } + })), + leases: jobs.map(job => { + const seeded = this.leases.get(`${job.topic}:${job.peerId}:${job.jobId}`) + const base = seeded ?? { + scope: this.scopeOf(job), + topic: job.topic as string, + peerId: job.peerId as string, + jobId: job.jobId as string, + leaseToken: decodeMongoUint64(job.leaseToken as string), + expiresAtMs: String((job.leaseUntil as Date).getTime()), + chainEpoch: decodeMongoUint64(job.chainEpoch as string), + topicHistoryGeneration: decodeMongoUint64(job.topicHistoryGeneration as string) + } + return { + ...base, + topicHistoryGeneration: decodeMongoUint64(job.topicHistoryGeneration as string) + } + }), + outputs: outputs.map(item => { + const outpoint = { txid: item.txid as string, outputIndex: item.outputIndex as string } + const topic = item.topic as string + const scope = this.scopeOf(item) + return { + key: outputKey(scope, topic, outpoint), + version: item.version as string, + ...(item.spender === undefined ? {} : { spentBy: item.spender as string }), + topic, + ...(item.state === 'unspent' + ? { + output: { + txid: item.txid as string, + outputIndex: item.outputIndex as string, + satoshis: decodeMongoUint64(item.satoshis as string), + score: decodeMongoUint64(item.score as string), + script: { + payload: { + digest: '00'.repeat(32), + byteLength: '0', + kind: 'locking-script' as const + }, + offset: decodeMongoUint64(item.scriptOffset as string), + byteLength: decodeMongoUint64(item.scriptByteLength as string) + } + } + } + : {}) + } + }), + edges: edges + .map(item => + edgeKey( + this.scopeOf(item), + item.topic as string, + { txid: item.sourceTxid as string, outputIndex: item.sourceOutputIndex as string }, + { txid: item.consumerTxid as string, outputIndex: item.consumerOutputIndex as string } + ) + ) + .sort(), + applied: applied.map(item => ({ + key: appliedKey(this.scopeOf(item), item.topic as string, item.txid as string), + record: { txid: item.txid as string } + })), + outbox: [ + ...lookup.map(item => ({ scope: this.scopeOf(item), eventId: item.eventId as string })), + ...propagation.map(item => ({ scope: this.scopeOf(item), eventId: item.eventId as string })) + ], + handoffs: jobs + .filter(item => typeof item.checkpoint === 'string' && item.checkpoint.length > 0) + .map(item => ({ topic: item.topic as string, checkpoint: item.checkpoint as string })), + historyUpdates, + operations: [] + } + } + + get adapter(): MongoAdmissionStorage { + return this.inner + } + + private scopeOf(document: { + network?: unknown + genesisHash?: unknown + nodeId?: unknown + }): StorageScope { + return { + network: document.network as string, + genesisHash: document.genesisHash as string, + nodeId: document.nodeId as string + } + } + + private async insertReadyPayload(ref: AdmissionPayloadRef): Promise { + const now = new Date() + const id = mongoRecordKey(mongoChainKey(this.fixture.scope), ref.kind, ref.digest) + try { + await this.fixture.db.collection(MongoCollectionNames.payloads).insertOne({ + _id: id, + schemaVersion: 1, + network: this.fixture.scope.network, + genesisHash: this.fixture.scope.genesisHash, + kind: ref.kind, + digest: ref.digest, + byteLength: encodeMongoUint64(ref.byteLength), + state: 'ready', + guard: randomUUID(), + ownerNodeId: this.fixture.scope.nodeId, + ownerId: randomUUID(), + fencingToken: encodeMongoUint64('1'), + leaseUntil: now, + createdAt: now, + updatedAt: now + }) + } catch (error) { + if (typeof error === 'object' && error !== null && 'code' in error && error.code === 11000) + return + throw error + } + } + + private async insertSpendable( + scope: StorageScope, + topic: string, + outpoint: AdmissionOutpoint, + version: string + ): Promise { + const now = new Date() + const id = mongoRecordKey( + mongoNodeKey(scope), + 'output', + topic, + outpoint.txid, + encodeMongoOutputIndex(outpoint.outputIndex) + ) + try { + await this.fixture.db.collection(MongoCollectionNames.outputs).insertOne({ + _id: id, + schemaVersion: 1, + network: scope.network, + genesisHash: scope.genesisHash, + nodeId: scope.nodeId, + topic, + txid: outpoint.txid, + outputIndex: encodeMongoOutputIndex(outpoint.outputIndex), + satoshis: encodeMongoUint64('1'), + score: encodeMongoUint64('1'), + scriptPayloadId: mongoRecordKey(mongoChainKey(scope), 'locking-script', 'bb'.repeat(32)), + scriptOffset: encodeMongoUint64('0'), + scriptByteLength: encodeMongoUint64('1'), + state: 'unspent', + version, + createdAt: now, + updatedAt: now + }) + } catch (error) { + if (typeof error === 'object' && error !== null && 'code' in error && error.code === 11000) { + await this.fixture.db + .collection(MongoCollectionNames.outputs) + .updateOne({ _id: id }, { $set: { version, state: 'unspent', updatedAt: now } }) + return + } + throw error + } + } + + private async insertHistory( + scope: StorageScope, + topic: string, + fence: { chainEpoch: string; topicHistoryGeneration: string } + ): Promise { + const now = new Date() + const id = mongoRecordKey(mongoNodeKey(scope), 'generation', topic) + await this.fixture.db.collection(MongoCollectionNames.topicGenerations).updateOne( + { _id: id }, + { + $setOnInsert: { + _id: id, + schemaVersion: 1, + network: scope.network, + genesisHash: scope.genesisHash, + nodeId: scope.nodeId, + topic, + policyId: 'policy-1', + createdAt: now + }, + $set: { + chainEpoch: encodeMongoUint64(fence.chainEpoch), + topicHistoryGeneration: encodeMongoUint64(fence.topicHistoryGeneration), + updatedAt: now + } + }, + { upsert: true } + ) + } + + private async insertLease(lease: RecoveryLease): Promise { + const now = new Date() + const id = mongoRecordKey( + mongoNodeKey(lease.scope), + 'job', + lease.topic, + lease.peerId, + lease.jobId + ) + await this.fixture.db.collection(MongoCollectionNames.basmRecoveryJobs).updateOne( + { _id: id }, + { + $setOnInsert: { + _id: id, + schemaVersion: 1, + network: lease.scope.network, + genesisHash: lease.scope.genesisHash, + nodeId: lease.scope.nodeId, + topic: lease.topic, + peerId: lease.peerId, + jobId: lease.jobId, + createdAt: now + }, + $set: { + chainEpoch: encodeMongoUint64(lease.chainEpoch), + topicHistoryGeneration: encodeMongoUint64(lease.topicHistoryGeneration), + leaseToken: encodeMongoUint64(lease.leaseToken), + leaseUntil: new Date(Date.now() + 60 * 60 * 1000), + state: 'leased', + checkpoint: lease.jobId, + updatedAt: now + } + }, + { upsert: true } + ) + } +} diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts new file mode 100644 index 000000000..c3458f9fd --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts @@ -0,0 +1,239 @@ +import { admissionPlan, admissionStorageContract } from '../admission/AdmissionStorageContract.js' +import { getAdmissionStorage } from '../../storage/AdmissionStorage.js' +import { MongoAdmissionStorage } from '../../storage/mongo/MongoAdmissionStorage.js' +import { MongoOverlayStorage } from '../../storage/mongo/MongoOverlayStorage.js' +import { bootstrapMongoOverlay, MongoCollectionNames } from '../../storage/mongo/MongoSchema.js' +import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' +import { MongoAdmissionHarness } from './MongoAdmissionHarness.js' +import type { MongoEnlistedLookupIndex } from '../../storage/mongo/MongoAdmissionStorage.js' +import type { AdmissionCommit } from '../../storage/AdmissionStorage.js' + +describe('Mongo admission storage', () => { + let fixture: MongoReplicaFixture + let harness: MongoAdmissionHarness + + beforeAll(async () => { + fixture = await createMongoReplicaFixture() + await bootstrapMongoOverlay(fixture.db, fixture.scope) + harness = new MongoAdmissionHarness(fixture) + }, 120000) + + afterAll(async () => { + await fixture.close() + }, 60000) + + admissionStorageContract(() => harness) + + test('advertises overlay-admission-v1 only on the complete adapter', () => { + const storage = new MongoOverlayStorage(fixture.db, fixture.scope) + expect(getAdmissionStorage(storage)?.protocol).toBe('overlay-admission-v1') + expect(getAdmissionStorage(storage.admission)).toBeUndefined() + }) + + test('lost ACK after majority commit returns the saved STEAK without a new body', async () => { + const plan = admissionPlan('lost-ack') + await harness.reset() + for (const decision of plan.decisions) { + await harness.seed.history(plan.identity.scope, decision.topic, decision.expectedHistory) + for (const read of decision.reads) { + if (read.expectedVersion !== null) { + await harness.seed.read( + plan.identity.scope, + decision.topic, + read.key, + read.expectedVersion + ) + } + } + for (const spend of decision.spends) { + await harness.seed.spendable( + plan.identity.scope, + decision.topic, + spend.outpoint, + spend.expectedVersion + ) + } + } + for (const payload of plan.payloads) await harness.seed.readyPayload(payload) + const first = await harness.adapter.commitAdmission(plan) + expect(first.state).toBe('committed') + const replacement = new MongoAdmissionStorage(fixture.db, fixture.scope) + let bodies = 0 + const original = replacement.commitAdmission.bind(replacement) + replacement.commitAdmission = async (next: AdmissionCommit) => { + bodies += 1 + return await original(next) + } + const retried = await replacement.commitAdmission(plan) + expect(retried).toEqual(first) + expect(bodies).toBe(1) + await replacement.close() + }) + + test('enlisted indexes are visible at commit and external indexes stay pending', async () => { + await harness.reset() + const visible: string[] = [] + const enlisted: MongoEnlistedLookupIndex = { + protocol: 'overlay-mongo-index-v1', + target: 'ls_enlisted', + apply: async (context, plan) => { + visible.push(plan.identity.txid) + await fixture.db + .collection('enlisted_index') + .insertOne({ txid: plan.identity.txid, target: 'ls_enlisted' }, context.options()) + } + } + const adapter = new MongoAdmissionStorage(fixture.db, fixture.scope, { + enlistedIndexes: [enlisted] + }) + const plan = admissionPlan('enlisted-visible', 'e1'.repeat(32)) + plan.outbox = plan.outbox.filter(intent => intent.kind !== 'lookup') + for (const decision of plan.decisions) { + await harness.seed.history(plan.identity.scope, decision.topic, decision.expectedHistory) + for (const read of decision.reads) { + if (read.expectedVersion !== null) { + await harness.seed.read( + plan.identity.scope, + decision.topic, + read.key, + read.expectedVersion + ) + } + } + for (const spend of decision.spends) { + await harness.seed.spendable( + plan.identity.scope, + decision.topic, + spend.outpoint, + spend.expectedVersion + ) + } + } + for (const payload of plan.payloads) await harness.seed.readyPayload(payload) + const result = await adapter.commitAdmission(plan) + expect(result.state).toBe('committed') + if (result.state !== 'committed') throw new Error('expected commit') + expect(result.receipt.indexes).toEqual([{ target: 'ls_enlisted', state: 'visible' }]) + expect( + await fixture.db.collection('enlisted_index').findOne({ txid: plan.identity.txid }) + ).toEqual(expect.objectContaining({ target: 'ls_enlisted' })) + expect(visible).toEqual([plan.identity.txid]) + await adapter.close() + }) + + test('external lookup index is not visible at commit and duplicate delivery is idempotent', async () => { + const plan = admissionPlan('outbox-delivery') + await harness.reset() + for (const decision of plan.decisions) { + await harness.seed.history(plan.identity.scope, decision.topic, decision.expectedHistory) + for (const read of decision.reads) { + if (read.expectedVersion !== null) { + await harness.seed.read( + plan.identity.scope, + decision.topic, + read.key, + read.expectedVersion + ) + } + } + for (const spend of decision.spends) { + await harness.seed.spendable( + plan.identity.scope, + decision.topic, + spend.outpoint, + spend.expectedVersion + ) + } + } + for (const payload of plan.payloads) await harness.seed.readyPayload(payload) + const result = await harness.adapter.commitAdmission(plan) + expect(result.state).toBe('committed') + if (result.state !== 'committed') throw new Error('expected commit') + expect(result.receipt.indexes).toEqual([{ target: 'ls_contract', state: 'pending' }]) + expect( + await fixture.db.collection('enlisted_index').findOne({ txid: plan.identity.txid }) + ).toBeNull() + const first = await harness.adapter.claimOutbox('lookup') + expect(first?.eventId).toBe(`${plan.key.operationId}:lookup`) + await harness.adapter.acknowledgeOutbox('lookup', first!.eventId) + await harness.adapter.acknowledgeOutbox('lookup', first!.eventId) + expect(await harness.adapter.claimOutbox('lookup')).toBeNull() + const stored = await fixture.db.collection(MongoCollectionNames.lookupOutbox).findOne({ + eventId: first!.eventId + }) + expect(stored?.state).toBe('delivered') + }) + + test('competing spend of a different spender is rejected', async () => { + await harness.reset() + const first = admissionPlan('race-a', '6'.repeat(64)) + const second = admissionPlan('race-b', '7'.repeat(64)) + for (const plan of [first, second]) { + for (const decision of plan.decisions) { + await harness.seed.history(plan.identity.scope, decision.topic, decision.expectedHistory) + for (const read of decision.reads) { + if (read.expectedVersion !== null) { + await harness.seed.read( + plan.identity.scope, + decision.topic, + read.key, + read.expectedVersion + ) + } + } + for (const spend of decision.spends) { + await harness.seed.spendable( + plan.identity.scope, + decision.topic, + spend.outpoint, + spend.expectedVersion + ) + } + } + for (const payload of plan.payloads) await harness.seed.readyPayload(payload) + } + const results = await Promise.all([ + harness.adapter.commitAdmission(first), + harness.adapter.commitAdmission(second) + ]) + expect(results.filter(result => result.state === 'committed')).toHaveLength(1) + expect( + results.filter(result => result.state === 'rejected' && result.code === 'spend-conflict') + ).toHaveLength(1) + }) + + test('majority admission effects survive killing the acknowledged primary', async () => { + const plan = admissionPlan('kill-primary') + await harness.reset() + for (const decision of plan.decisions) { + await harness.seed.history(plan.identity.scope, decision.topic, decision.expectedHistory) + for (const read of decision.reads) { + if (read.expectedVersion !== null) { + await harness.seed.read( + plan.identity.scope, + decision.topic, + read.key, + read.expectedVersion + ) + } + } + for (const spend of decision.spends) { + await harness.seed.spendable( + plan.identity.scope, + decision.topic, + spend.outpoint, + spend.expectedVersion + ) + } + } + for (const payload of plan.payloads) await harness.seed.readyPayload(payload) + const committed = await harness.adapter.commitAdmission(plan) + expect(committed.state).toBe('committed') + await fixture.killPrimary() + const client = await fixture.connect() + fixture.db = client.db(fixture.db.databaseName) + const replacement = new MongoAdmissionStorage(fixture.db, fixture.scope) + expect(await replacement.commitAdmission(plan)).toEqual(committed) + await replacement.close() + }, 60000) +}) diff --git a/packages/overlays/overlay/src/storage/mongo.ts b/packages/overlays/overlay/src/storage/mongo.ts index d73a111ef..a64a93056 100644 --- a/packages/overlays/overlay/src/storage/mongo.ts +++ b/packages/overlays/overlay/src/storage/mongo.ts @@ -3,3 +3,6 @@ export * from './mongo/MongoSchema.js' export * from './mongo/MongoPayloadStore.js' export * from './mongo/MongoReadGuards.js' export * from './mongo/MongoTransactionRunner.js' +export * from './mongo/MongoAdmissionPlan.js' +export * from './mongo/MongoAdmissionStorage.js' +export * from './mongo/MongoOverlayStorage.js' diff --git a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts new file mode 100644 index 000000000..218dd5055 --- /dev/null +++ b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts @@ -0,0 +1,274 @@ +import { + admissionSemanticDigest, + parseStorageOutputIndex, + parseStorageUint64, + type AdmissionCommit, + type AdmissionCommitResult, + type AdmissionOutboxIntent, + type AdmissionOutpoint, + type AdmissionPayloadRef, + type AdmissionReceipt, + type StorageScope +} from '../AdmissionStorage.js' + +export type AdmissionRejectionCode = Extract['code'] + +export class AdmissionRejectedError extends Error { + readonly code: AdmissionRejectionCode + + constructor(code: AdmissionRejectionCode) { + super(code) + this.name = 'AdmissionRejectedError' + this.code = code + } +} + +export const rejectAdmission = (code: AdmissionRejectionCode): never => { + throw new AdmissionRejectedError(code) +} + +const isHash = (value: string): boolean => /^[0-9a-f]{64}$/.test(value) + +const isUint64 = (value: string): boolean => { + try { + parseStorageUint64(value) + return true + } catch { + return false + } +} + +const isWireOutpoint = (outpoint: AdmissionOutpoint): boolean => { + if (!isHash(outpoint.txid)) return false + try { + parseStorageOutputIndex(outpoint.outputIndex) + return true + } catch { + return false + } +} + +export const sameScope = (left: StorageScope, right: StorageScope): boolean => + left.network === right.network && + left.genesisHash === right.genesisHash && + left.nodeId === right.nodeId + +export const samePayload = ( + left: AdmissionPayloadRef | undefined, + right: AdmissionPayloadRef +): boolean => + left !== undefined && + left.digest === right.digest && + left.byteLength === right.byteLength && + left.kind === right.kind + +export const admissionPlanPayloads = (plan: AdmissionCommit): AdmissionPayloadRef[] => { + const references = [...plan.payloads] + for (const decision of plan.decisions) { + for (const output of decision.outputs) references.push(output.script.payload) + if (decision.applied.proof !== undefined) references.push(decision.applied.proof) + } + for (const intent of plan.outbox) references.push(...intent.payloads) + return references +} + +const steakAdmittedIndexes = (plan: AdmissionCommit, topic: string): number[] | undefined => { + let steak: unknown + try { + steak = JSON.parse(plan.steak) + } catch { + return undefined + } + if (typeof steak !== 'object' || steak === null || Array.isArray(steak)) return undefined + const record = steak as Record + if (Object.keys(record).length !== plan.decisions.length) return undefined + const entry = record[topic] + if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) return undefined + const result = entry as Record + if (!Array.isArray(result.outputsToAdmit)) return undefined + if (result.coinsToRetain !== undefined && !Array.isArray(result.coinsToRetain)) return undefined + if (result.coinsRemoved !== undefined && !Array.isArray(result.coinsRemoved)) return undefined + if ( + result.outputsToAdmit.some( + value => + typeof value !== 'number' || !Number.isInteger(value) || value < 0 || value > 4294967295 + ) + ) { + return undefined + } + return result.outputsToAdmit as number[] +} + +export const isBoundSteak = (plan: AdmissionCommit): boolean => { + if (typeof plan.steak !== 'string' || !plan.steak.isWellFormed()) return false + try { + JSON.parse(plan.steak) + } catch { + return false + } + return plan.decisions.every(decision => { + const admitted = steakAdmittedIndexes(plan, decision.topic) + if (admitted === undefined) return false + try { + const expected = decision.outputs.map(output => parseStorageOutputIndex(output.outputIndex)) + return JSON.stringify(admitted) === JSON.stringify(expected) + } catch { + return false + } + }) +} + +export const validateAdmissionPlan = ( + plan: AdmissionCommit +): AdmissionRejectionCode | undefined => { + let semanticDigest: string + try { + semanticDigest = admissionSemanticDigest(plan.identity) + } catch { + return 'digest-mismatch' + } + if ( + plan.key.semanticDigest !== semanticDigest || + !sameScope(plan.key.scope, plan.identity.scope) + ) { + return 'digest-mismatch' + } + const topics = new Set(plan.identity.topics.map(item => item.topic)) + const decisionTopics = new Set(plan.decisions.map(decision => decision.topic)) + if ( + topics.size !== plan.identity.topics.length || + plan.decisions.length !== topics.size || + decisionTopics.size !== plan.decisions.length || + decisionTopics.size !== topics.size || + plan.decisions.some(decision => !topics.has(decision.topic)) || + new Set(plan.outbox.map(intent => intent.eventId)).size !== plan.outbox.length + ) { + return 'invalid-plan' + } + if ( + plan.identity.mode === 'historical' && + plan.outbox.some(intent => intent.kind === 'propagation') + ) { + return 'invalid-plan' + } + const payloads = admissionPlanPayloads(plan) + if ( + payloads.some( + ref => !isHash(ref.digest) || !isUint64(ref.byteLength) || ref.digest.length === 0 + ) + ) { + return 'payload-not-ready' + } + if (!isBoundSteak(plan)) return 'invalid-plan' + for (const intent of plan.outbox) { + if ( + intent.eventId.length === 0 || + !intent.eventId.isWellFormed() || + intent.target.length === 0 || + !intent.target.isWellFormed() || + (intent.kind !== 'lookup' && intent.kind !== 'propagation') + ) { + return 'invalid-plan' + } + } + for (const decision of plan.decisions) { + if ( + !isUint64(decision.expectedHistory.chainEpoch) || + !isUint64(decision.expectedHistory.topicHistoryGeneration) + ) { + return 'invalid-plan' + } + if ( + decision.spends.some( + spend => !isWireOutpoint(spend.outpoint) || spend.spender !== plan.identity.txid + ) + ) { + return 'invalid-plan' + } + if (decision.evictions.some(eviction => !isWireOutpoint(eviction))) return 'invalid-plan' + if ( + decision.outputs.some(output => { + if (output.txid !== plan.identity.txid || !isWireOutpoint(output)) return true + if ( + ![output.satoshis, output.score, output.script.offset, output.script.byteLength].every( + isUint64 + ) + ) { + return true + } + if (!isHash(output.script.payload.digest) || !isUint64(output.script.payload.byteLength)) + return true + return ( + parseStorageUint64(output.script.offset) + parseStorageUint64(output.script.byteLength) > + parseStorageUint64(output.script.payload.byteLength) + ) + }) + ) { + return 'invalid-plan' + } + if ( + decision.edges.some(edge => !isWireOutpoint(edge.source) || !isWireOutpoint(edge.consumer)) + ) { + return 'invalid-plan' + } + const applied = decision.applied + if (applied.txid !== plan.identity.txid || !isHash(applied.txid)) return 'invalid-plan' + if (applied.firstSeenHeight !== undefined && !isUint64(applied.firstSeenHeight)) + return 'invalid-plan' + if ( + applied.proof !== undefined && + (!isHash(applied.proof.digest) || !isUint64(applied.proof.byteLength)) + ) { + return 'invalid-plan' + } + if ( + applied.block !== undefined && + (![applied.block.height, applied.block.index].every(isUint64) || + !isHash(applied.block.hash) || + !isHash(applied.block.merkleRoot)) + ) { + return 'invalid-plan' + } + if (decision.historyUpdate !== undefined) { + if ( + !isUint64(decision.historyUpdate.nextTopicHistoryGeneration) || + !isUint64(decision.historyUpdate.affectedFromHeight) || + parseStorageUint64(decision.historyUpdate.nextTopicHistoryGeneration) <= + parseStorageUint64(decision.expectedHistory.topicHistoryGeneration) + ) { + return 'invalid-plan' + } + } + } + return undefined +} + +export const admissionReceiptFor = ( + plan: AdmissionCommit, + enlistedTargets: readonly string[] +): AdmissionReceipt => { + const enlisted = new Set(enlistedTargets) + const lookupTargets = [ + ...new Set(plan.outbox.filter(intent => intent.kind === 'lookup').map(intent => intent.target)) + ] + if (lookupTargets.some(target => enlisted.has(target))) rejectAdmission('invalid-plan') + return { + operationId: plan.key.operationId, + semanticDigest: plan.key.semanticDigest, + durability: 'atomic-local', + steak: plan.steak, + indexes: [ + ...enlistedTargets.map(target => ({ target, state: 'visible' as const })), + ...lookupTargets.map(target => ({ target, state: 'pending' as const })) + ], + propagation: plan.outbox.some(intent => intent.kind === 'propagation') + ? 'pending' + : 'not-requested' + } +} + +export const lookupOutboxIntents = (plan: AdmissionCommit): AdmissionOutboxIntent[] => + plan.outbox.filter(intent => intent.kind === 'lookup') + +export const propagationOutboxIntents = (plan: AdmissionCommit): AdmissionOutboxIntent[] => + plan.outbox.filter(intent => intent.kind === 'propagation') diff --git a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts new file mode 100644 index 000000000..8271757ce --- /dev/null +++ b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts @@ -0,0 +1,936 @@ +import { createHash } from 'node:crypto' +import { setTimeout as delay } from 'node:timers/promises' +import type { Collection, Db, Document } from 'mongodb' +import { + admissionSemanticDigest, + isReplaySafeProjection, + type AdmissionCommit, + type AdmissionCommitResult, + type AdmissionOperationKey, + type AdmissionOutboxIntent, + type AdmissionOutpoint, + type AdmissionPayloadRef, + type AdmissionReconcileResult, + type AdmissionStorage, + type AdmissionTopicDecision, + type StorageScope +} from '../AdmissionStorage.js' +import type { RecoveryLease } from '../RecoveryContract.js' +import { + AdmissionRejectedError, + admissionPlanPayloads, + admissionReceiptFor, + lookupOutboxIntents, + propagationOutboxIntents, + rejectAdmission, + sameScope, + validateAdmissionPlan +} from './MongoAdmissionPlan.js' +import { MongoPayloadStore, type MongoPayloadKind } from './MongoPayloadStore.js' +import { MongoReadGuardConflictError, MongoReadGuards } from './MongoReadGuards.js' +import { + MongoCollectionNames, + encodeMongoOutputIndex, + encodeMongoUint64, + decodeMongoUint64, + mongoChainKey, + mongoNodeKey, + mongoRecordKey +} from './MongoSchema.js' +import { + MongoTransactionRunner, + type MongoTransactionContext, + type MongoTransactionOptions +} from './MongoTransactionRunner.js' + +export interface MongoEnlistedLookupIndex { + readonly protocol: 'overlay-mongo-index-v1' + readonly target: string + apply: (context: MongoTransactionContext, plan: AdmissionCommit) => Promise +} + +export interface MongoAdmissionStorageOptions { + runner?: MongoTransactionRunner + payloads?: MongoPayloadStore + readGuards?: MongoReadGuards + enlistedIndexes?: MongoEnlistedLookupIndex[] + projector?: unknown +} + +export interface MongoOutboxLease { + eventId: string + kind: 'lookup' | 'propagation' + target: string + payloads: AdmissionPayloadRef[] +} + +interface OutputDocument extends Document { + _id: string + topic: string + txid: string + outputIndex: string + satoshis: string + score: string + scriptPayloadId: string + scriptOffset: string + scriptByteLength: string + state: 'unspent' | 'spent' | 'evicted' + spender?: string + version: string +} + +interface AppliedDocument extends Document { + _id: string + topic: string + txid: string + state: 'active' | 'unproven' | 'evicted' + admissionId: string + firstSeenHeight?: string + proofPayloadId?: string +} + +interface GenerationDocument extends Document { + _id: string + topic: string + chainEpoch: string + topicHistoryGeneration: string + policyId: string +} + +interface JobDocument extends Document { + _id: string + topic: string + peerId: string + jobId: string + chainEpoch: string + topicHistoryGeneration: string + leaseToken: string + leaseUntil: Date + state: string + checkpoint: string +} + +interface OutboxDocument extends Document { + _id: string + eventId: string + target: string + state: string + nextAttemptAt?: Date + leaseUntil?: Date +} + +interface IdDocument extends Document { + _id: string + [key: string]: unknown +} + +const majority = { w: 'majority' as const, j: true } +const schemaVersion = 1 + +const duplicateKey = (error: unknown): boolean => + typeof error === 'object' && error !== null && 'code' in error && error.code === 11000 + +const isPayloadKind = (value: string): value is MongoPayloadKind => + value === 'raw-transaction' || + value === 'merkle-path' || + value === 'beef-manifest' || + value === 'locking-script' || + value === 'outbox-data' + +/** + * Opt-in Mongo AdmissionStorage. Engine submit uses this only when + * getAdmissionStorage(storage) observes overlay-admission-v1. + */ +export class MongoAdmissionStorage implements AdmissionStorage { + readonly protocol = 'overlay-admission-v1' as const + readonly scope: StorageScope + private readonly runner: MongoTransactionRunner + private readonly payloads: MongoPayloadStore + private readonly guards: MongoReadGuards + private readonly enlisted: MongoEnlistedLookupIndex[] + private readonly projector: unknown + private readonly peers = new Map() + + constructor( + private readonly db: Db, + scope: StorageScope, + options: MongoAdmissionStorageOptions = {} + ) { + this.scope = { ...scope } + this.runner = options.runner ?? new MongoTransactionRunner(db, this.scope) + this.payloads = options.payloads ?? new MongoPayloadStore(db, this.scope) + this.guards = options.readGuards ?? new MongoReadGuards(db) + this.enlisted = [...(options.enlistedIndexes ?? [])] + this.projector = options.projector + if ( + this.enlisted.some( + index => index.protocol !== 'overlay-mongo-index-v1' || index.target.length === 0 + ) + ) { + throw new Error('Invalid enlisted Mongo lookup index') + } + if (new Set(this.enlisted.map(index => index.target)).size !== this.enlisted.length) { + throw new Error('Duplicate enlisted Mongo lookup index target') + } + } + + enlistedTargets(): string[] { + return this.enlisted.map(index => index.target) + } + + async commitAdmission(plan: AdmissionCommit): Promise { + if (!sameScope(plan.key.scope, this.scope) || !sameScope(plan.identity.scope, this.scope)) { + return await this.peerFor(plan.key.scope).commitAdmission(plan) + } + let semanticDigest: string + try { + semanticDigest = admissionSemanticDigest(plan.identity) + } catch { + return { state: 'rejected', code: 'digest-mismatch' } + } + if ( + plan.key.semanticDigest !== semanticDigest || + !sameScope(plan.key.scope, plan.identity.scope) + ) { + return { state: 'rejected', code: 'digest-mismatch' } + } + if (this.projector !== undefined && !isReplaySafeProjection(this.projector)) { + return { state: 'rejected', code: 'unsupported-projection' } + } + const existing = await this.runner.reconcile(plan.key) + if (existing.state === 'committed' || existing.state === 'rejected') return existing + if (existing.state === 'pending' && existing.attemptId !== 'unlocated') { + const waited = await this.waitForPending(plan.key, existing.attemptId) + if (waited.state === 'committed' || waited.state === 'rejected') return waited + } + const rejected = validateAdmissionPlan(plan) + if (rejected !== undefined) return { state: 'rejected', code: rejected } + let receipt + try { + receipt = admissionReceiptFor(plan, this.enlistedTargets()) + } catch (error) { + return this.asResult(error) + } + await this.prepareReadGuards(plan) + await this.publishHistoryUpdatePayloads(plan) + try { + const result = await this.runner.run( + { key: plan.key, identity: plan.identity, receipt }, + async context => { + await this.applyPlan(context, plan) + } + ) + if (result.state !== 'pending') return result + return await this.waitForPending(plan.key, result.attemptId) + } catch (error) { + return this.asResult(error) + } + } + + private async waitForPending( + key: AdmissionOperationKey, + attemptId: string + ): Promise { + let current: AdmissionCommitResult = { state: 'pending', attemptId } + for (let index = 0; index < 50 && current.state === 'pending'; index += 1) { + await delay(20) + const reconciled = await this.runner.reconcile( + key, + current.state === 'pending' ? current.attemptId : attemptId + ) + if (reconciled.state === 'committed' || reconciled.state === 'rejected') return reconciled + if (reconciled.state === 'aborted') { + return { state: 'pending', attemptId } + } + current = reconciled + } + return current + } + + async reconcileAdmission( + key: AdmissionOperationKey, + attemptId?: string, + options: MongoTransactionOptions = {} + ): Promise { + if (!sameScope(key.scope, this.scope)) + return await this.peerFor(key.scope).reconcileAdmission(key, attemptId, options) + return await this.runner.reconcile(key, attemptId, options) + } + + async close(): Promise { + await Promise.all([ + this.runner.close(), + ...[...this.peers.values()].map(async peer => await peer.close()) + ]) + } + + private peerFor(scope: StorageScope): MongoAdmissionStorage { + const id = mongoNodeKey(scope) + const existing = this.peers.get(id) + if (existing !== undefined) return existing + const peer = new MongoAdmissionStorage(this.db, scope, { + enlistedIndexes: this.enlisted, + projector: this.projector + }) + this.peers.set(id, peer) + return peer + } + + async claimOutbox( + kind: 'lookup' | 'propagation', + leaseMS = 15_000 + ): Promise { + if (!Number.isSafeInteger(leaseMS) || leaseMS < 1 || leaseMS > 60_000) { + throw new Error('Invalid Mongo outbox lease') + } + const claimed = await this.outbox(kind).findOneAndUpdate( + { + network: this.scope.network, + genesisHash: this.scope.genesisHash, + nodeId: this.scope.nodeId, + $or: [{ state: 'pending' }, { state: 'leased', $expr: { $lte: ['$leaseUntil', '$$NOW'] } }] + }, + [ + { + $set: { + state: 'leased', + leaseUntil: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: leaseMS } }, + nextAttemptAt: { + $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: leaseMS } + }, + updatedAt: '$$NOW' + } + } + ], + { returnDocument: 'after', writeConcern: majority } + ) + if (claimed === null) return null + return { + eventId: claimed.eventId, + kind, + target: claimed.target, + payloads: await this.outboxPayloads(kind, claimed.eventId) + } + } + + async acknowledgeOutbox(kind: 'lookup' | 'propagation', eventId: string): Promise { + const result = await this.outbox(kind).updateOne( + { + _id: this.outboxId(kind, eventId), + eventId, + state: { $in: ['leased', 'delivered'] } + }, + { $set: { state: 'delivered', updatedAt: new Date() } }, + { writeConcern: majority } + ) + if (result.matchedCount !== 1) throw new Error('Mongo outbox event is not leased') + } + + private asResult(error: unknown): AdmissionCommitResult { + if (error instanceof AdmissionRejectedError) return { state: 'rejected', code: error.code } + if (error instanceof MongoReadGuardConflictError) + return { state: 'rejected', code: 'read-conflict' } + if (error instanceof Error && error.message === 'Mongo payload is not ready for reference') { + return { state: 'rejected', code: 'payload-not-ready' } + } + throw error + } + + private async prepareReadGuards(plan: AdmissionCommit): Promise { + for (const decision of plan.decisions) { + for (const read of decision.reads) await this.guards.initialize(this.scope, read.key) + } + } + + private async publishHistoryUpdatePayloads(plan: AdmissionCommit): Promise { + for (const decision of plan.decisions) { + if (decision.historyUpdate === undefined) continue + const bytes = Buffer.from(JSON.stringify(this.historyUpdateRecord(decision)), 'utf8') + const digest = createHash('sha256').update(bytes).digest('hex') + await this.payloads.publish({ + kind: 'outbox-data', + digest, + byteLength: String(bytes.byteLength), + bytes: (async function* () { + yield bytes + })() + }) + } + } + + private historyUpdateRecord(decision: AdmissionTopicDecision): { + affectedFromHeight: string + checkpoint?: string + } { + if (decision.historyUpdate === undefined) return { affectedFromHeight: '0' } + return { + affectedFromHeight: decision.historyUpdate.affectedFromHeight, + ...(decision.historyUpdate.handoff === undefined + ? {} + : { checkpoint: decision.historyUpdate.handoff.checkpoint }) + } + } + + private async applyPlan(context: MongoTransactionContext, plan: AdmissionCommit): Promise { + await this.assertReadyPayloads(context, plan) + for (const decision of plan.decisions) { + await this.checkReads(context, decision) + await this.checkHistory(context, decision) + await this.assertAppliedAvailable(context, plan, decision) + } + for (const intent of plan.outbox) await this.assertOutboxAvailable(context, intent) + for (const decision of plan.decisions) { + for (const spend of decision.spends) await this.applySpend(context, decision.topic, spend) + for (const eviction of decision.evictions) + await this.applyEviction(context, decision.topic, eviction) + for (const output of decision.outputs) await this.insertOutput(context, decision, output) + for (const edge of decision.edges) await this.insertEdge(context, decision.topic, edge) + await this.insertApplied(context, plan, decision) + await this.applyHistoryUpdate( + context, + decision, + plan.identity.topics.find(item => item.topic === decision.topic)?.policyId ?? 'default' + ) + } + await this.upsertTransaction(context, plan) + for (const ref of admissionPlanPayloads(plan)) { + await this.pin(context, ref, 'transaction', plan.identity.txid, `${ref.kind}:${ref.digest}`) + } + for (const intent of lookupOutboxIntents(plan)) + await this.insertOutbox(context, 'lookup', intent) + for (const intent of propagationOutboxIntents(plan)) { + await this.insertOutbox(context, 'propagation', intent) + } + for (const index of this.enlisted) await index.apply(context, plan) + } + + private async assertReadyPayloads( + context: MongoTransactionContext, + plan: AdmissionCommit + ): Promise { + const seen = new Set() + for (const ref of admissionPlanPayloads(plan)) { + const id = this.payloadId(ref) + if (seen.has(id)) continue + seen.add(id) + const found = await this.db + .collection(MongoCollectionNames.payloads) + .findOne({ _id: id, state: 'ready', digest: ref.digest, kind: ref.kind }, context.options()) + if (found === null || decodeMongoUint64(String(found.byteLength)) !== ref.byteLength) { + rejectAdmission('payload-not-ready') + } + } + } + + private async checkReads( + context: MongoTransactionContext, + decision: AdmissionTopicDecision + ): Promise { + for (const read of decision.reads) { + const options = context.options() + await this.guards.check( + options.session, + { scope: this.scope, key: read.key, expectedVersion: read.expectedVersion }, + { timeoutMS: options.timeoutMS, signal: context.signal } + ) + } + } + + private async checkHistory( + context: MongoTransactionContext, + decision: AdmissionTopicDecision + ): Promise { + const document = await this.generations().findOne( + { _id: this.generationId(decision.topic) }, + context.options() + ) + const epoch = document === null ? '0' : decodeMongoUint64(document.chainEpoch) + const generation = document === null ? '0' : decodeMongoUint64(document.topicHistoryGeneration) + if ( + epoch !== decision.expectedHistory.chainEpoch || + generation !== decision.expectedHistory.topicHistoryGeneration + ) { + rejectAdmission('read-conflict') + } + } + + private async assertAppliedAvailable( + context: MongoTransactionContext, + plan: AdmissionCommit, + decision: AdmissionTopicDecision + ): Promise { + const existing = await this.applied().findOne( + { _id: this.appliedId(decision.topic, decision.applied.txid) }, + context.options() + ) + if (existing !== null && existing.admissionId !== plan.key.operationId) { + rejectAdmission('invalid-plan') + } + } + + private async assertOutboxAvailable( + context: MongoTransactionContext, + intent: AdmissionOutboxIntent + ): Promise { + const existing = await this.outbox(intent.kind).findOne( + { _id: this.outboxId(intent.kind, intent.eventId) }, + context.options() + ) + if (existing !== null) rejectAdmission('invalid-plan') + } + + private async applySpend( + context: MongoTransactionContext, + topic: string, + spend: { outpoint: AdmissionOutpoint; expectedVersion: string; spender: string } + ): Promise { + const id = this.outputId(topic, spend.outpoint) + const result = await this.outputs().findOneAndUpdate( + { + _id: id, + version: spend.expectedVersion, + state: 'unspent' + }, + { + $set: { + state: 'spent', + spender: spend.spender, + updatedAt: new Date() + } + }, + { ...context.options(), returnDocument: 'after' } + ) + if (result === null) { + const current = await this.outputs().findOne({ _id: id }, context.options()) + if (current?.spender === spend.spender && current.state === 'spent') return + rejectAdmission('spend-conflict') + } + } + + private async applyEviction( + context: MongoTransactionContext, + topic: string, + eviction: AdmissionOutpoint + ): Promise { + const id = this.outputId(topic, eviction) + const result = await this.outputs().updateOne( + { _id: id }, + { $set: { state: 'evicted', updatedAt: new Date() } }, + context.options() + ) + if (result.matchedCount !== 1) rejectAdmission('invalid-plan') + } + + private async insertOutput( + context: MongoTransactionContext, + decision: AdmissionTopicDecision, + output: AdmissionTopicDecision['outputs'][number] + ): Promise { + const now = new Date() + const payloadId = this.payloadId(output.script.payload) + try { + await this.outputs().insertOne( + { + _id: this.outputId(decision.topic, output), + schemaVersion, + network: this.scope.network, + genesisHash: this.scope.genesisHash, + nodeId: this.scope.nodeId, + topic: decision.topic, + txid: output.txid, + outputIndex: encodeMongoOutputIndex(output.outputIndex), + satoshis: encodeMongoUint64(output.satoshis), + score: encodeMongoUint64(output.score), + scriptPayloadId: payloadId, + scriptOffset: encodeMongoUint64(output.script.offset), + scriptByteLength: encodeMongoUint64(output.script.byteLength), + state: 'unspent', + version: '1', + createdAt: now, + updatedAt: now + }, + context.options() + ) + } catch (error) { + if (duplicateKey(error)) rejectAdmission('invalid-plan') + throw error + } + await this.pin( + context, + output.script.payload, + 'output', + this.outputId(decision.topic, output), + 'script' + ) + } + + private async insertEdge( + context: MongoTransactionContext, + topic: string, + edge: { source: AdmissionOutpoint; consumer: AdmissionOutpoint } + ): Promise { + const now = new Date() + const id = mongoRecordKey( + mongoNodeKey(this.scope), + 'edge', + topic, + edge.source.txid, + encodeMongoOutputIndex(edge.source.outputIndex), + edge.consumer.txid, + encodeMongoOutputIndex(edge.consumer.outputIndex) + ) + await this.db.collection(MongoCollectionNames.consumptionEdges).updateOne( + { _id: id }, + { + $setOnInsert: { + _id: id, + schemaVersion, + network: this.scope.network, + genesisHash: this.scope.genesisHash, + nodeId: this.scope.nodeId, + topic, + sourceTxid: edge.source.txid, + sourceOutputIndex: encodeMongoOutputIndex(edge.source.outputIndex), + consumerTxid: edge.consumer.txid, + consumerOutputIndex: encodeMongoOutputIndex(edge.consumer.outputIndex), + createdAt: now + }, + $set: { updatedAt: now } + }, + { upsert: true, ...context.options() } + ) + } + + private async insertApplied( + context: MongoTransactionContext, + plan: AdmissionCommit, + decision: AdmissionTopicDecision + ): Promise { + const now = new Date() + const proven = decision.applied.block !== undefined + try { + await this.applied().insertOne( + { + _id: this.appliedId(decision.topic, decision.applied.txid), + schemaVersion, + network: this.scope.network, + genesisHash: this.scope.genesisHash, + nodeId: this.scope.nodeId, + topic: decision.topic, + txid: decision.applied.txid, + state: proven ? 'active' : 'unproven', + admissionId: plan.key.operationId, + ...(decision.applied.firstSeenHeight === undefined + ? {} + : { firstSeenHeight: encodeMongoUint64(decision.applied.firstSeenHeight) }), + ...(decision.applied.proof === undefined + ? {} + : { proofPayloadId: this.payloadId(decision.applied.proof) }), + createdAt: now, + updatedAt: now + }, + context.options() + ) + } catch (error) { + if (duplicateKey(error)) rejectAdmission('invalid-plan') + throw error + } + if (decision.applied.proof !== undefined) { + await this.pin( + context, + decision.applied.proof, + 'applied-history', + this.appliedId(decision.topic, decision.applied.txid), + 'proof' + ) + } + } + + private async applyHistoryUpdate( + context: MongoTransactionContext, + decision: AdmissionTopicDecision, + policyId: string + ): Promise { + if (decision.historyUpdate === undefined) return + const now = new Date() + const next = decision.historyUpdate.nextTopicHistoryGeneration + const updated = await this.generations().findOneAndUpdate( + { + _id: this.generationId(decision.topic), + chainEpoch: encodeMongoUint64(decision.expectedHistory.chainEpoch), + topicHistoryGeneration: encodeMongoUint64(decision.expectedHistory.topicHistoryGeneration) + }, + { + $set: { + topicHistoryGeneration: encodeMongoUint64(next), + policyId, + updatedAt: now + } + }, + { ...context.options(), returnDocument: 'after' } + ) + if (updated === null) { + if ( + decision.expectedHistory.chainEpoch !== '0' || + decision.expectedHistory.topicHistoryGeneration !== '0' + ) { + rejectAdmission('read-conflict') + } + try { + await this.generations().insertOne( + { + _id: this.generationId(decision.topic), + schemaVersion, + network: this.scope.network, + genesisHash: this.scope.genesisHash, + nodeId: this.scope.nodeId, + topic: decision.topic, + chainEpoch: encodeMongoUint64(decision.expectedHistory.chainEpoch), + topicHistoryGeneration: encodeMongoUint64(next), + policyId, + createdAt: now, + updatedAt: now + }, + context.options() + ) + } catch (error) { + if (duplicateKey(error)) rejectAdmission('read-conflict') + throw error + } + } + if (decision.historyUpdate.handoff !== undefined) { + await this.applyHandoff(context, decision, decision.historyUpdate.handoff.expected, next) + } + const bytes = Buffer.from(JSON.stringify(this.historyUpdateRecord(decision)), 'utf8') + const digest = createHash('sha256').update(bytes).digest('hex') + const ref: AdmissionPayloadRef = { + kind: 'outbox-data', + digest, + byteLength: String(bytes.byteLength) + } + const payload = await this.db + .collection(MongoCollectionNames.payloads) + .findOne({ _id: this.payloadId(ref), state: 'ready' }, context.options()) + if (payload === null) rejectAdmission('payload-not-ready') + await this.pin(context, ref, 'basm-job', decision.topic, 'history-update') + } + + private async applyHandoff( + context: MongoTransactionContext, + decision: AdmissionTopicDecision, + expected: RecoveryLease, + nextGeneration: string + ): Promise { + if ( + !sameScope(expected.scope, this.scope) || + expected.topic !== decision.topic || + expected.chainEpoch !== decision.expectedHistory.chainEpoch || + expected.topicHistoryGeneration !== decision.expectedHistory.topicHistoryGeneration + ) { + rejectAdmission('read-conflict') + } + const now = new Date() + const result = await this.jobs().findOneAndUpdate( + { + _id: this.jobId(expected), + topic: expected.topic, + peerId: expected.peerId, + jobId: expected.jobId, + chainEpoch: encodeMongoUint64(expected.chainEpoch), + topicHistoryGeneration: encodeMongoUint64(expected.topicHistoryGeneration), + leaseToken: encodeMongoUint64(expected.leaseToken), + $expr: { $gt: ['$leaseUntil', '$$NOW'] } + }, + { + $set: { + topicHistoryGeneration: encodeMongoUint64(nextGeneration), + checkpoint: decision.historyUpdate?.handoff?.checkpoint ?? expected.jobId, + updatedAt: now + } + }, + { ...context.options(), returnDocument: 'after' } + ) + if (result === null) rejectAdmission('read-conflict') + } + + private async upsertTransaction( + context: MongoTransactionContext, + plan: AdmissionCommit + ): Promise { + const raw = plan.payloads.find(ref => ref.kind === 'raw-transaction') + const manifest = plan.payloads.find(ref => ref.kind === 'beef-manifest') + const now = new Date() + const id = mongoRecordKey(mongoChainKey(this.scope), 'transaction', plan.identity.txid) + await this.db.collection(MongoCollectionNames.transactions).updateOne( + { _id: id }, + { + $setOnInsert: { + _id: id, + schemaVersion, + network: this.scope.network, + genesisHash: this.scope.genesisHash, + txid: plan.identity.txid, + ...(raw === undefined ? {} : { rawPayloadId: this.payloadId(raw) }), + ...(manifest === undefined ? {} : { manifestPayloadId: this.payloadId(manifest) }), + createdAt: now + }, + $set: { updatedAt: now } + }, + { upsert: true, ...context.options() } + ) + } + + private async insertOutbox( + context: MongoTransactionContext, + kind: 'lookup' | 'propagation', + intent: AdmissionOutboxIntent + ): Promise { + const now = new Date() + try { + await this.outbox(kind).insertOne( + { + _id: this.outboxId(kind, intent.eventId), + schemaVersion, + network: this.scope.network, + genesisHash: this.scope.genesisHash, + nodeId: this.scope.nodeId, + eventId: intent.eventId, + target: intent.target, + state: 'pending', + nextAttemptAt: now, + leaseUntil: new Date(0), + createdAt: now, + updatedAt: now + }, + context.options() + ) + } catch (error) { + if (duplicateKey(error)) rejectAdmission('invalid-plan') + throw error + } + for (const [index, payload] of intent.payloads.entries()) { + await this.pin(context, payload, `${kind}-outbox`, intent.eventId, String(index)) + } + } + + private async pin( + context: MongoTransactionContext, + ref: AdmissionPayloadRef, + ownerKind: + | 'transaction' + | 'applied-history' + | 'output' + | 'gasp-graph' + | 'gasp-node' + | 'basm-job' + | 'lookup-outbox' + | 'propagation-outbox' + | 'manifest' + | 'pin', + ownerId: string, + slot: string + ): Promise { + if (!isPayloadKind(ref.kind)) rejectAdmission('invalid-plan') + const options = context.options() + try { + await this.payloads.addReference( + options.session, + { + scope: this.scope, + payload: { kind: ref.kind, digest: ref.digest }, + ownerKind, + ownerId, + slot + }, + { timeoutMS: options.timeoutMS, signal: context.signal } + ) + } catch (error) { + if (error instanceof Error && error.message === 'Mongo payload is not ready for reference') { + rejectAdmission('payload-not-ready') + } + throw error + } + } + + private async outboxPayloads( + kind: 'lookup' | 'propagation', + eventId: string + ): Promise { + const refs = await this.db + .collection(MongoCollectionNames.payloadReferences) + .find({ + network: this.scope.network, + genesisHash: this.scope.genesisHash, + nodeId: this.scope.nodeId, + ownerKind: `${kind}-outbox`, + ownerId: eventId + }) + .sort({ slot: 1 }) + .toArray() + const result: AdmissionPayloadRef[] = [] + for (const reference of refs) { + const payload = await this.db + .collection(MongoCollectionNames.payloads) + .findOne({ _id: String(reference.payloadId) }) + if (payload === null || !isPayloadKind(payload.kind as string)) continue + result.push({ + kind: payload.kind as MongoPayloadKind, + digest: payload.digest as string, + byteLength: decodeMongoUint64(payload.byteLength as string) + }) + } + return result + } + + payloadId(ref: Pick): string { + return mongoRecordKey(mongoChainKey(this.scope), ref.kind, ref.digest) + } + + outputId(topic: string, outpoint: AdmissionOutpoint): string { + return mongoRecordKey( + mongoNodeKey(this.scope), + 'output', + topic, + outpoint.txid, + encodeMongoOutputIndex(outpoint.outputIndex) + ) + } + + private appliedId(topic: string, txid: string): string { + return mongoRecordKey(mongoNodeKey(this.scope), 'applied', topic, txid) + } + + generationId(topic: string): string { + return mongoRecordKey(mongoNodeKey(this.scope), 'generation', topic) + } + + jobId(lease: Pick): string { + return mongoRecordKey(mongoNodeKey(this.scope), 'job', lease.topic, lease.peerId, lease.jobId) + } + + outboxId(kind: 'lookup' | 'propagation', eventId: string): string { + return mongoRecordKey(mongoNodeKey(this.scope), `${kind}-outbox`, eventId) + } + + private outputs(): Collection { + return this.db.collection(MongoCollectionNames.outputs) + } + + private applied(): Collection { + return this.db.collection(MongoCollectionNames.appliedTransactions) + } + + generations(): Collection { + return this.db.collection(MongoCollectionNames.topicGenerations) + } + + jobs(): Collection { + return this.db.collection(MongoCollectionNames.basmRecoveryJobs) + } + + outbox(kind: 'lookup' | 'propagation'): Collection { + return this.db.collection( + kind === 'lookup' ? MongoCollectionNames.lookupOutbox : MongoCollectionNames.propagationOutbox + ) + } +} diff --git a/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts new file mode 100644 index 000000000..a34305fd2 --- /dev/null +++ b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts @@ -0,0 +1,351 @@ +import { createHash } from 'node:crypto' +import type { Db, Document } from 'mongodb' +import type { Output } from '../../Output.js' +import type { AppliedTransaction, Storage } from '../Storage.js' +import type { AdmissionPayloadRef, HistoryFence, StorageScope } from '../AdmissionStorage.js' +import { + MongoAdmissionStorage, + type MongoAdmissionStorageOptions +} from './MongoAdmissionStorage.js' +import { MongoPayloadStore, type MongoPayloadKind } from './MongoPayloadStore.js' +import { + MongoCollectionNames, + decodeMongoUint64, + encodeMongoOutputIndex, + encodeMongoUint64, + mongoNodeKey, + mongoRecordKey +} from './MongoSchema.js' + +type IdDocument = Document & { _id: string } + +const isPayloadKind = (value: string): value is MongoPayloadKind => + value === 'raw-transaction' || + value === 'merkle-path' || + value === 'beef-manifest' || + value === 'locking-script' || + value === 'outbox-data' + +/** + * Opt-in Mongo Storage adapter that advertises overlay-admission-v1 only because + * commitAdmission actually honors majority ack, spends, history, and outbox. + */ +export class MongoOverlayStorage implements Storage { + readonly admission: MongoAdmissionStorage + readonly admissionScope: StorageScope + private readonly payloads: MongoPayloadStore + + constructor( + private readonly db: Db, + scope: StorageScope, + options: MongoAdmissionStorageOptions = {} + ) { + this.admissionScope = { ...scope } + this.payloads = options.payloads ?? new MongoPayloadStore(db, this.admissionScope) + this.admission = new MongoAdmissionStorage(db, this.admissionScope, { + ...options, + payloads: this.payloads + }) + } + + enlistedIndexTargets(): readonly string[] { + return this.admission.enlistedTargets() + } + + async getHistoryFence(topic: string): Promise { + const document = await this.admission.generations().findOne({ + _id: this.admission.generationId(topic) + }) + if (document === null) return { chainEpoch: '0', topicHistoryGeneration: '0' } + return { + chainEpoch: decodeMongoUint64(document.chainEpoch), + topicHistoryGeneration: decodeMongoUint64(document.topicHistoryGeneration) + } + } + + async publishAdmissionPayload(input: { + kind: AdmissionPayloadRef['kind'] + bytes: Uint8Array + txid?: string + }): Promise { + if (!isPayloadKind(input.kind)) throw new Error('Invalid Mongo admission payload kind') + const digest = createHash('sha256').update(input.bytes).digest('hex') + const published = await this.payloads.publish({ + kind: input.kind, + digest, + byteLength: String(input.bytes.byteLength), + txid: input.txid, + bytes: (async function* () { + yield input.bytes + })() + }) + return { + kind: published.kind, + digest: published.digest, + byteLength: published.byteLength + } + } + + async close(): Promise { + await this.admission.close() + } + + async insertOutput(utxo: Output): Promise { + const now = new Date() + const script = Buffer.from(utxo.outputScript) + const payload = await this.publishAdmissionPayload({ + kind: 'locking-script', + bytes: script + }) + await this.db.collection(MongoCollectionNames.outputs).updateOne( + { + _id: this.outputId(utxo.topic, utxo.txid, utxo.outputIndex), + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId + }, + { + $setOnInsert: { + _id: this.outputId(utxo.topic, utxo.txid, utxo.outputIndex), + schemaVersion: 1, + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId, + topic: utxo.topic, + txid: utxo.txid, + outputIndex: encodeMongoOutputIndex(String(utxo.outputIndex)), + satoshis: encodeMongoUint64(String(utxo.satoshis)), + score: encodeMongoUint64(String(utxo.score ?? 0)), + scriptPayloadId: this.admission.payloadId(payload), + scriptOffset: encodeMongoUint64('0'), + scriptByteLength: encodeMongoUint64(payload.byteLength), + state: utxo.spent ? 'spent' : 'unspent', + version: '1', + createdAt: now + }, + $set: { updatedAt: now } + }, + { upsert: true, writeConcern: { w: 'majority', j: true } } + ) + } + + async findOutput( + txid: string, + outputIndex: number, + topic?: string, + spent?: boolean + ): Promise { + const filter: Record = { + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId, + txid, + outputIndex: encodeMongoOutputIndex(String(outputIndex)) + } + if (topic !== undefined) filter.topic = topic + if (spent === true) filter.state = 'spent' + if (spent === false) filter.state = 'unspent' + const document = await this.db + .collection(MongoCollectionNames.outputs) + .findOne(filter, { + readConcern: { level: 'majority' }, + readPreference: 'primary' + }) + if (document === null) return null + return this.toOutput(document) + } + + async findOutputsForTransaction(txid: string): Promise { + const documents = await this.db + .collection(MongoCollectionNames.outputs) + .find({ + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId, + txid + }) + .toArray() + return documents.map(document => this.toOutput(document)) + } + + async findUTXOsForTopic(topic: string, since?: number, limit?: number): Promise { + const filter: Record = { + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId, + topic, + state: 'unspent' + } + if (since !== undefined && since > 0) filter.score = { $gte: encodeMongoUint64(String(since)) } + let query = this.db + .collection(MongoCollectionNames.outputs) + .find(filter) + .sort({ score: 1, _id: 1 }) + if (limit !== undefined && limit > 0) query = query.limit(limit) + return (await query.toArray()).map(document => this.toOutput(document)) + } + + async deleteOutput(txid: string, outputIndex: number, topic: string): Promise { + await this.db + .collection(MongoCollectionNames.outputs) + .updateOne( + { _id: this.outputId(topic, txid, outputIndex) }, + { $set: { state: 'evicted', updatedAt: new Date() } }, + { writeConcern: { w: 'majority', j: true } } + ) + } + + async markUTXOAsSpent(txid: string, outputIndex: number, topic: string): Promise { + await this.db + .collection(MongoCollectionNames.outputs) + .updateOne( + { _id: this.outputId(topic, txid, outputIndex), state: 'unspent' }, + { $set: { state: 'spent', updatedAt: new Date() } }, + { writeConcern: { w: 'majority', j: true } } + ) + } + + async updateConsumedBy( + txid: string, + outputIndex: number, + topic: string, + consumedBy: Array<{ txid: string; outputIndex: number }> + ): Promise { + const now = new Date() + for (const consumer of consumedBy) { + const id = mongoRecordKey( + mongoNodeKey(this.admissionScope), + 'edge', + topic, + txid, + encodeMongoOutputIndex(String(outputIndex)), + consumer.txid, + encodeMongoOutputIndex(String(consumer.outputIndex)) + ) + await this.db.collection(MongoCollectionNames.consumptionEdges).updateOne( + { _id: id }, + { + $setOnInsert: { + _id: id, + schemaVersion: 1, + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId, + topic, + sourceTxid: txid, + sourceOutputIndex: encodeMongoOutputIndex(String(outputIndex)), + consumerTxid: consumer.txid, + consumerOutputIndex: encodeMongoOutputIndex(String(consumer.outputIndex)), + createdAt: now + }, + $set: { updatedAt: now } + }, + { upsert: true, writeConcern: { w: 'majority', j: true } } + ) + } + } + + async updateTransactionBEEF(_txid: string, _beef: number[]): Promise {} + + async insertAppliedTransaction(tx: AppliedTransaction): Promise { + const now = new Date() + const id = mongoRecordKey(mongoNodeKey(this.admissionScope), 'applied', tx.topic, tx.txid) + await this.db.collection(MongoCollectionNames.appliedTransactions).updateOne( + { _id: id }, + { + $setOnInsert: { + _id: id, + schemaVersion: 1, + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId, + topic: tx.topic, + txid: tx.txid, + state: tx.proven === true ? 'active' : 'unproven', + admissionId: tx.txid, + createdAt: now + }, + $set: { updatedAt: now } + }, + { upsert: true, writeConcern: { w: 'majority', j: true } } + ) + } + + async doesAppliedTransactionExist(tx: AppliedTransaction): Promise { + const found = await this.db + .collection(MongoCollectionNames.appliedTransactions) + .findOne( + { + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId, + topic: tx.topic, + txid: tx.txid + }, + { readConcern: { level: 'majority' }, readPreference: 'primary' } + ) + return found !== null + } + + async updateLastInteraction(host: string, topic: string, since: number): Promise { + const now = new Date() + const id = mongoRecordKey(mongoNodeKey(this.admissionScope), 'cursor', host, topic) + await this.db.collection(MongoCollectionNames.gaspCursors).updateOne( + { _id: id }, + { + $setOnInsert: { + _id: id, + schemaVersion: 1, + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId, + remoteHost: host, + topic, + createdAt: now + }, + $set: { + state: 'ready', + score: encodeMongoUint64(String(since)), + updatedAt: now + } + }, + { upsert: true, writeConcern: { w: 'majority', j: true } } + ) + } + + async getLastInteraction(host: string, topic: string): Promise { + const found = await this.db.collection(MongoCollectionNames.gaspCursors).findOne({ + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId, + remoteHost: host, + topic + }) + if (found?.score === undefined) return 0 + return Number(decodeMongoUint64(found.score as string)) + } + + private outputId(topic: string, txid: string, outputIndex: number): string { + return mongoRecordKey( + mongoNodeKey(this.admissionScope), + 'output', + topic, + txid, + encodeMongoOutputIndex(String(outputIndex)) + ) + } + + private toOutput(document: Record): Output { + return { + txid: document.txid as string, + outputIndex: Number(document.outputIndex), + outputScript: [], + satoshis: Number(decodeMongoUint64(document.satoshis as string)), + topic: document.topic as string, + spent: document.state === 'spent', + outputsConsumed: [], + consumedBy: [], + score: Number(decodeMongoUint64(document.score as string)) + } + } +} diff --git a/specs/overlay/mongo-v1.md b/specs/overlay/mongo-v1.md index 93d0a0f32..eef01aa71 100644 --- a/specs/overlay/mongo-v1.md +++ b/specs/overlay/mongo-v1.md @@ -5,9 +5,9 @@ This is an additive, opt-in MongoDB storage foundation for `@bsv/overlay`. It supplies schema bootstrap, scoped keys, content-addressed payload primitives, and durable read-guard primitives. It does **not** select MongoDB by default, -connect MongoDB to `Engine`, implement `AdmissionStorage`, or make an admission -receipt claim. The existing Engine, Knex, and injected-storage paths retain -their current behavior. +change the SQL default, or claim production activation. `MongoOverlayStorage` +may be injected explicitly; Knex and injected-storage callers without that +adapter retain their current behavior. MongoDB is an optional peer dependency. An application that imports a Mongo entry point must install a compatible driver explicitly: @@ -91,29 +91,30 @@ controls where they operate in a caller's admission body. Neither payload publication nor collection runs verifier logic, network activity, uploads, or plugin callbacks inside an admission transaction. -## Admission transaction integration is pending - -The foundation is deliberately smaller than the v1 admission contract. A later -Engine/adapter integration must compose the guarded payload primitives with the -whole admission operation in one short transaction. Until that integration and -its conformance evidence land, no Mongo method is an `AdmissionStorage` -implementation and no saved STEAK/receipt is exposed as a durable admission -acknowledgment. - -The pending transaction core must use a finite retry budget and distinguish -MongoDB's labels on the **same session and operation identity**: rerun a -`TransientTransactionError` body only under that identity, and reconcile an -`UnknownTransactionCommitResult` before a new body may run. Its completion -record must be majority+journal persisted with the exact receipt. Expired -orphan cleanup must use a compare-and-set proof rather than treating absence or -an ambiguous commit as an abort. MongoDB documents the retry-label behavior in -its [transaction API guidance](https://www.mongodb.com/docs/manual/core/transactions-in-applications/). - -That integration must keep external indexes and propagation asynchronous: -persist an idempotent outbox intent in the admission transaction, then let a -leased worker deliver it. A callback may participate only after it declares -replay safety, reconciliation, cancellation, and bounded work. It must never -be invoked in the Mongo admission transaction. +## Admission transaction integration + +`MongoAdmissionStorage` implements `overlay-admission-v1` using the schema, +payload, read-guard, and transaction-runner primitives. `MongoOverlayStorage` +is the Engine `Storage` adapter that advertises that capability only because +`commitAdmission` actually honors majority ack, conditional spends, applied +history, and outbox publication. `getAdmissionStorage(storage)` remains a +declaration check: Knex and incomplete Mongo helpers still return undefined. + +Engine `submit` / historical submit call `commitAdmission` when that capability +is present. Parse, SPV, topic decision, and broadcast stay outside the Mongo +transaction. Broadcast-before-mutation ordering is unchanged for live mode; +historical mode still skips broadcast/propagation. The returned STEAK is the +exact UTF-8 JSON saved with the majority receipt. A crash retry of the same +operation identity returns that saved STEAK. `TransientTransactionError` +reruns the body under the same operation id; `UnknownTransactionCommitResult` +reconciles the same commit identity before a new body. + +Native Mongo lookup indexes may enlist writes on the admission session and +are the only indexes that may be `visible` at commit. External plugins receive +a durable lookup outbox event with leased delivery; they are not assumed +replay-safe, and STEAK is overlay admission, not remote delivery. Propagation +uses the same outbox protocol. Ban/serving eviction must not erase historical +applied/BASM records. Mongo is not the overlay default. ## Compatibility and operations diff --git a/specs/overlay/persistence-v1.md b/specs/overlay/persistence-v1.md index aa4bae016..272a07210 100644 --- a/specs/overlay/persistence-v1.md +++ b/specs/overlay/persistence-v1.md @@ -21,7 +21,8 @@ declarations only. Go uses a separate optional provider/interface, preserving it existing `engine.Storage` implementers. Shape detection cannot certify durability. The existing engines, SQL adapter, injected Go stores and default selections are unchanged. Their existing STEAK callback must not be described as this receipt. -S03/S04 must integrate and verify the capability before activating durable ACKs. +S03 integrates the TS Engine/Mongo capability behind an explicit adapter; +durable ACKs are not a production default. S04 owns the Go equivalent. ## Identity and exact values From 5d1eeed57bb1249b2af9cefbb9815444f56bc660 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 17:34:06 -0500 Subject: [PATCH 014/168] fix(sdk): encode overlay discovery cache keys with stringifyBRC100 LookupResolver is a BRC-100 JSON boundary file, so the in-flight discovery key cannot use JSON.stringify. stringifyBRC100 preserves the same service-plus-limit tuple without changing cache cover semantics. --- .../sdk/src/overlay-tools/LookupResolver.ts | 2 +- .../__tests/LookupResolver.dynamic.test.ts | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/overlay-tools/LookupResolver.ts b/packages/sdk/src/overlay-tools/LookupResolver.ts index e38f6338b..2d1ccb985 100644 --- a/packages/sdk/src/overlay-tools/LookupResolver.ts +++ b/packages/sdk/src/overlay-tools/LookupResolver.ts @@ -384,7 +384,7 @@ function lookupDiscoveryBound(limits: LookupLimits): LookupDiscoveryBound { /** In-flight discovery identity: service plus the limits that shape tracker work. */ function lookupDiscoveryCacheKey(service: string, limits: LookupLimits): string { const bound = lookupDiscoveryBound(limits) - return JSON.stringify([ + return stringifyBRC100([ service, bound.maxHosts, bound.maxHostsPerTracker, diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts index 760b1ecd9..ac2959eb6 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts @@ -884,6 +884,49 @@ describe('LookupResolver dynamic discovery', () => { ) }) + it('does not coalesce in-flight discovery across distinct caller limits', async () => { + const tracker = 'https://inflight-key-tracker.example' + const host = 'https://inflight-key-host.example' + const service = 'ls_inflight_key' + const receipt = await slapReceipt(180, host, service) + let finishTracker: (() => void) | undefined + const trackerGate = new Promise(resolve => { + finishTracker = resolve + }) + const lookup = jest.fn(async (url: string) => { + if (url === tracker) { + await trackerGate + return { type: 'output-list' as const, outputs: [receipt] } + } + return { type: 'output-list' as const, outputs: [] } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + const first = resolver.query$({ service, query: { n: 1 } }, undefined, { + limits: { maxHosts: 1 } + })[Symbol.asyncIterator]() + const firstPending = first.next() + await Promise.resolve() + const second = resolver.query$({ service, query: { n: 2 } }, undefined, { + limits: { maxHosts: 2 } + })[Symbol.asyncIterator]() + const secondPending = second.next() + await Promise.resolve() + + expect((resolver as any).hostsInFlight.size).toBe(2) + const keys = Array.from((resolver as any).hostsInFlight.keys()) as string[] + expect(keys).toHaveLength(2) + expect(keys[0]).not.toEqual(keys[1]) + expect(keys.every(key => key.includes(service))).toBe(true) + expect(lookup.mock.calls.filter(([url]) => url === tracker)).toHaveLength(2) + + finishTracker?.() + await jest.runAllTimersAsync() + await firstPending + await secondPending + await first.return?.() + await second.return?.() + }) + it('throws from query() when a deadline expires before any host is admitted', async () => { const tracker = 'https://deadline-miss-tracker.example' const lookup = jest.fn( From f96148202e3db8cb3927ff583d12484b6bd7e943 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 17:53:13 -0500 Subject: [PATCH 015/168] fix(overlay): close S03 admission STEAK, history pin, and hydration gaps Keep failed-topic STEAK keys and merkle applied metadata on the Engine admission plan. Version history-update pins by generation and hydrate Mongo output scripts and BEEF without unsafe integer coercion. --- packages/overlays/overlay/src/Engine.ts | 4 +- .../overlays/overlay/src/EngineAdmission.ts | 64 ++++++- .../src/__tests/Engine.admission.test.ts | 34 ++++ .../__tests/mongo/MongoAdmissionHarness.ts | 2 +- .../mongo/MongoAdmissionStorage.test.ts | 97 ++++++++++ .../src/storage/mongo/MongoAdmissionPlan.ts | 47 ++--- .../storage/mongo/MongoAdmissionStorage.ts | 62 ++++++- .../src/storage/mongo/MongoOverlayStorage.ts | 166 ++++++++++++++++-- .../src/storage/mongo/MongoPayloadStore.ts | 45 +++++ 9 files changed, 474 insertions(+), 47 deletions(-) diff --git a/packages/overlays/overlay/src/Engine.ts b/packages/overlays/overlay/src/Engine.ts index 492db54a6..0e8a87411 100644 --- a/packages/overlays/overlay/src/Engine.ts +++ b/packages/overlays/overlay/src/Engine.ts @@ -1133,6 +1133,7 @@ export class Engine { this.advertiser !== undefined && mode !== 'historical-tx' && mode !== 'historical-tx-no-spv' + const applied = await this.buildAppliedTransactionRecord(tx) const buildPlan = async () => await buildOverlayAdmissionPlan({ host: admissionHost, @@ -1145,7 +1146,8 @@ export class Engine { validations, failedTopics, lookupServices: this.lookupServices, - includePropagation + includePropagation, + applied }) const committed = await waitForAdmissionReceipt( admissionHost.admission, diff --git a/packages/overlays/overlay/src/EngineAdmission.ts b/packages/overlays/overlay/src/EngineAdmission.ts index 4832bc619..0f2e81fab 100644 --- a/packages/overlays/overlay/src/EngineAdmission.ts +++ b/packages/overlays/overlay/src/EngineAdmission.ts @@ -1,5 +1,6 @@ import { createHash } from 'node:crypto' import type { STEAK, Transaction } from '@bsv/sdk' +import { extractMerkleProofMetadata } from './BASM.js' import { admissionSemanticDigest, getAdmissionStorage, @@ -118,6 +119,13 @@ export async function buildOverlayAdmissionPlan(input: { failedTopics: Set lookupServices: { [key: string]: LookupService } includePropagation: boolean + applied?: { + firstSeenHeight?: number + blockHeight?: number + blockHash?: string + blockIndex?: number + merkleRoot?: string + } }): Promise { const accepted = input.validations.filter( validation => @@ -146,13 +154,25 @@ export async function buildOverlayAdmissionPlan(input: { input.txid ) const payloads: AdmissionPayloadRef[] = [raw] + let proof: AdmissionPayloadRef | undefined if (input.tx.merklePath !== undefined) { - payloads.push( - await localPayload(input.host, 'merkle-path', Buffer.from(input.tx.merklePath.toBinary())) + proof = await localPayload( + input.host, + 'merkle-path', + Buffer.from(input.tx.merklePath.toBinary()) ) + payloads.push(proof) } + const merkle = extractMerkleProofMetadata(input.txid, input.tx.merklePath) const decisions: AdmissionTopicDecision[] = [] const steak: STEAK = {} + for (const validation of input.validations) { + steak[validation.topic] = { + outputsToAdmit: validation.admissibleOutputs.outputsToAdmit, + coinsToRetain: validation.admissibleOutputs.coinsToRetain, + coinsRemoved: validation.admissibleOutputs.coinsRemoved ?? [] + } + } for (const validation of accepted) { const { outputsConsumed, outputsToMarkStale } = classifyCoins(input.tx, validation) const fence = @@ -201,7 +221,7 @@ export async function buildOverlayAdmissionPlan(input: { consumer: { txid: output.txid, outputIndex: output.outputIndex } })) ), - applied: { txid: input.txid } + applied: appliedRecord(input, proof, merkle) }) steak[validation.topic] = { outputsToAdmit: validation.admissibleOutputs.outputsToAdmit, @@ -272,6 +292,44 @@ export async function waitForAdmissionReceipt( throw new Error('Overlay admission commit is pending') } +function appliedRecord( + input: { + txid: string + applied?: { + firstSeenHeight?: number + blockHeight?: number + blockHash?: string + blockIndex?: number + merkleRoot?: string + } + }, + proof: AdmissionPayloadRef | undefined, + merkle: ReturnType +): AdmissionTopicDecision['applied'] { + const applied: AdmissionTopicDecision['applied'] = { txid: input.txid } + const firstSeen = input.applied?.firstSeenHeight ?? merkle?.blockHeight + if (firstSeen !== undefined) applied.firstSeenHeight = String(firstSeen) + if (proof !== undefined) applied.proof = proof + const blockHash = input.applied?.blockHash + const height = input.applied?.blockHeight ?? merkle?.blockHeight + const index = input.applied?.blockIndex ?? merkle?.blockIndex + const merkleRoot = input.applied?.merkleRoot ?? merkle?.merkleRoot + if ( + blockHash !== undefined && + height !== undefined && + index !== undefined && + merkleRoot !== undefined + ) { + applied.block = { + height: String(height), + hash: blockHash, + index: String(index), + merkleRoot + } + } + return applied +} + function uniquePayloads(payloads: AdmissionPayloadRef[]): AdmissionPayloadRef[] { const seen = new Set() const result: AdmissionPayloadRef[] = [] diff --git a/packages/overlays/overlay/src/__tests/Engine.admission.test.ts b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts index 6392d62dc..481c64de8 100644 --- a/packages/overlays/overlay/src/__tests/Engine.admission.test.ts +++ b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts @@ -202,4 +202,38 @@ describe('Engine admission submit', () => { }) expect(order).toEqual(['commit', 'callback']) }) + + test('keeps failed-topic STEAK keys and attaches merkle applied metadata', async () => { + const rejectManager: TopicManager = { + identifyAdmissibleOutputs: jest.fn(async () => { + throw new Error('topic rejected') + }), + getDocumentation: async () => 'docs', + getMetaData: async () => ({ name: 'Reject', shortDescription: 'Reject' }) + } + const proven = exampleTX.inputs[0].sourceTransaction + if (proven === undefined || proven.merklePath === undefined) { + throw new Error('expected the BEEF ancestor to carry a merkle path') + } + const engine = new Engine( + { Hello: mockTopicManager, Reject: rejectManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker + ) + engine.logger = { ...console, error: jest.fn() } + const steak = await engine.submit({ + beef: proven.toBEEF(), + topics: ['Hello', 'Reject'] + }) + expect(Object.keys(steak).sort()).toEqual(['Hello', 'Reject']) + expect(steak.Hello.outputsToAdmit).toEqual([0]) + expect(steak.Reject.outputsToAdmit).toEqual([]) + const plan = commitAdmission.mock.calls[0][0] as AdmissionCommit + expect(JSON.parse(plan.steak)).toEqual(steak) + expect(plan.decisions).toHaveLength(1) + expect(plan.decisions[0].applied.proof?.kind).toBe('merkle-path') + expect(plan.decisions[0].applied.firstSeenHeight).toBe('800000') + expect(plan.decisions[0].applied.block).toBeUndefined() + }) }) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts index db5f233dc..0e92737b3 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts @@ -207,7 +207,7 @@ export class MongoAdmissionHarness implements AdmissionStorageContractHarness { .collection(MongoCollectionNames.propagationOutbox) .find() .toArray() - const historyRefs = refs.filter(item => item.slot === 'history-update') + const historyRefs = refs.filter(item => String(item.slot).startsWith('history-update')) const historyUpdates = [] for (const reference of historyRefs) { const payload = payloads.find(item => item._id === reference.payloadId) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts index c3458f9fd..936f9695d 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts @@ -1,3 +1,4 @@ +import { Transaction } from '@bsv/sdk' import { admissionPlan, admissionStorageContract } from '../admission/AdmissionStorageContract.js' import { getAdmissionStorage } from '../../storage/AdmissionStorage.js' import { MongoAdmissionStorage } from '../../storage/mongo/MongoAdmissionStorage.js' @@ -202,6 +203,96 @@ describe('Mongo admission storage', () => { ).toHaveLength(1) }) + test('sequential history updates replace the generation-scoped pin', async () => { + await harness.reset() + const first = admissionPlan('history-first') + first.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '4', + affectedFromHeight: '99' + } + const second = admissionPlan('history-second', '6'.repeat(64)) + second.decisions[0].spends = [] + second.decisions[0].expectedHistory = { chainEpoch: '7', topicHistoryGeneration: '4' } + second.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '5', + affectedFromHeight: '100' + } + await harness.seed.history( + first.identity.scope, + first.decisions[0].topic, + first.decisions[0].expectedHistory + ) + for (const read of first.decisions[0].reads) { + if (read.expectedVersion !== null) { + await harness.seed.read( + first.identity.scope, + first.decisions[0].topic, + read.key, + read.expectedVersion + ) + } + } + for (const spend of first.decisions[0].spends) { + await harness.seed.spendable( + first.identity.scope, + first.decisions[0].topic, + spend.outpoint, + spend.expectedVersion + ) + } + for (const payload of first.payloads) await harness.seed.readyPayload(payload) + expect((await harness.adapter.commitAdmission(first)).state).toBe('committed') + await harness.seed.history( + second.identity.scope, + second.decisions[0].topic, + second.decisions[0].expectedHistory + ) + for (const read of second.decisions[0].reads) { + if (read.expectedVersion !== null) { + await harness.seed.read( + second.identity.scope, + second.decisions[0].topic, + read.key, + read.expectedVersion + ) + } + } + for (const payload of second.payloads) await harness.seed.readyPayload(payload) + expect((await harness.adapter.commitAdmission(second)).state).toBe('committed') + const snapshot = await harness.snapshot() + expect(snapshot.fences.find(item => item.topic === 'tm_contract')?.fence).toEqual({ + chainEpoch: '7', + topicHistoryGeneration: '5' + }) + expect(snapshot.historyUpdates).toEqual([{ topic: 'tm_contract', affectedFromHeight: '100' }]) + }) + + test('hydrates output scripts and BEEF from payload bytes', async () => { + await harness.reset() + const storage = new MongoOverlayStorage(fixture.db, fixture.scope) + const script = [0x76, 0xa9, 0x14, 0x00, 0x88, 0xac] + const beef = hydrationBeef() + const txid = Transaction.fromBEEF(beef).id('hex') + await storage.insertOutput({ + txid, + outputIndex: 0, + outputScript: script, + satoshis: 1234, + topic: 'tm_contract', + spent: false, + outputsConsumed: [], + consumedBy: [], + score: 10, + beef + }) + const found = await storage.findOutput(txid, 0, 'tm_contract', false, true) + expect(found?.outputScript).toEqual(script) + expect(found?.satoshis).toBe(1234) + expect(found?.score).toBe(10) + expect(found?.beef?.length).toBeGreaterThan(0) + await storage.close() + }) + test('majority admission effects survive killing the acknowledged primary', async () => { const plan = admissionPlan('kill-primary') await harness.reset() @@ -237,3 +328,9 @@ describe('Mongo admission storage', () => { await replacement.close() }, 60000) }) + +function hydrationBeef(): number[] { + return Transaction.fromHexBEEF( + '0100beef01fe636d0c0007021400fe507c0c7aa754cef1f7889d5fd395cf1f785dd7de98eed895dbedfe4e5bc70d1502ac4e164f5bc16746bb0868404292ac8318bbac3800e4aad13a014da427adce3e010b00bc4ff395efd11719b277694cface5aa50d085a0bb81f613f70313acd28cf4557010400574b2d9142b8d28b61d88e3b2c3f44d858411356b49a28a4643b6d1a6a092a5201030051a05fc84d531b5d250c23f4f886f6812f9fe3f402d61607f977b4ecd2701c19010000fd781529d58fc2523cf396a7f25440b409857e7e221766c57214b1d38c7b481f01010062f542f45ea3660f86c013ced80534cb5fd4c19d66c56e7e8c5d4bf2d40acc5e010100b121e91836fd7cd5102b654e9f72f3cf6fdbfd0b161c53a9c54b12c841126331020100000001cd4e4cac3c7b56920d1e7655e7e260d31f29d9a388d04910f1bbd72304a79029010000006b483045022100e75279a205a547c445719420aa3138bf14743e3f42618e5f86a19bde14bb95f7022064777d34776b05d816daf1699493fcdf2ef5a5ab1ad710d9c97bfb5b8f7cef3641210263e2dee22b1ddc5e11f6fab8bcd2378bdd19580d640501ea956ec0e786f93e76ffffffff013e660000000000001976a9146bfd5c7fbe21529d45803dbcf0c87dd3c71efbc288ac0000000001000100000001ac4e164f5bc16746bb0868404292ac8318bbac3800e4aad13a014da427adce3e000000006a47304402203a61a2e931612b4bda08d541cfb980885173b8dcf64a3471238ae7abcd368d6402204cbf24f04b9aa2256d8901f0ed97866603d2be8324c2bfb7a37bf8fc90edd5b441210263e2dee22b1ddc5e11f6fab8bcd2378bdd19580d640501ea956ec0e786f93e76ffffffff013c660000000000001976a9146bfd5c7fbe21529d45803dbcf0c87dd3c71efbc288ac0000000000' + ).toBEEF() +} diff --git a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts index 218dd5055..43769e50a 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts @@ -72,46 +72,37 @@ export const admissionPlanPayloads = (plan: AdmissionCommit): AdmissionPayloadRe return references } -const steakAdmittedIndexes = (plan: AdmissionCommit, topic: string): number[] | undefined => { - let steak: unknown - try { - steak = JSON.parse(plan.steak) - } catch { - return undefined - } - if (typeof steak !== 'object' || steak === null || Array.isArray(steak)) return undefined - const record = steak as Record - if (Object.keys(record).length !== plan.decisions.length) return undefined - const entry = record[topic] - if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) return undefined +const isSteakEntry = (entry: unknown): entry is { outputsToAdmit: number[] } => { + if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) return false const result = entry as Record - if (!Array.isArray(result.outputsToAdmit)) return undefined - if (result.coinsToRetain !== undefined && !Array.isArray(result.coinsToRetain)) return undefined - if (result.coinsRemoved !== undefined && !Array.isArray(result.coinsRemoved)) return undefined - if ( - result.outputsToAdmit.some( - value => - typeof value !== 'number' || !Number.isInteger(value) || value < 0 || value > 4294967295 - ) - ) { - return undefined - } - return result.outputsToAdmit as number[] + if (!Array.isArray(result.outputsToAdmit)) return false + if (result.coinsToRetain !== undefined && !Array.isArray(result.coinsToRetain)) return false + if (result.coinsRemoved !== undefined && !Array.isArray(result.coinsRemoved)) return false + return !result.outputsToAdmit.some( + value => + typeof value !== 'number' || !Number.isInteger(value) || value < 0 || value > 4294967295 + ) } export const isBoundSteak = (plan: AdmissionCommit): boolean => { if (typeof plan.steak !== 'string' || !plan.steak.isWellFormed()) return false + let steak: unknown try { - JSON.parse(plan.steak) + steak = JSON.parse(plan.steak) } catch { return false } + if (typeof steak !== 'object' || steak === null || Array.isArray(steak)) return false + const record = steak as Record + for (const entry of Object.values(record)) { + if (!isSteakEntry(entry)) return false + } return plan.decisions.every(decision => { - const admitted = steakAdmittedIndexes(plan, decision.topic) - if (admitted === undefined) return false + const entry = record[decision.topic] + if (!isSteakEntry(entry)) return false try { const expected = decision.outputs.map(output => parseStorageOutputIndex(output.outputIndex)) - return JSON.stringify(admitted) === JSON.stringify(expected) + return JSON.stringify(entry.outputsToAdmit) === JSON.stringify(expected) } catch { return false } diff --git a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts index 8271757ce..243ec89ea 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts @@ -712,7 +712,67 @@ export class MongoAdmissionStorage implements AdmissionStorage { .collection(MongoCollectionNames.payloads) .findOne({ _id: this.payloadId(ref), state: 'ready' }, context.options()) if (payload === null) rejectAdmission('payload-not-ready') - await this.pin(context, ref, 'basm-job', decision.topic, 'history-update') + await this.replaceHistoryPin( + context, + decision.topic, + decision.expectedHistory.topicHistoryGeneration, + next, + ref + ) + } + + private historyUpdateSlot(generation: string): string { + return `history-update:${generation}` + } + + private async replaceHistoryPin( + context: MongoTransactionContext, + topic: string, + previousGeneration: string, + nextGeneration: string, + ref: AdmissionPayloadRef + ): Promise { + const previousSlot = this.historyUpdateSlot(previousGeneration) + const previous = await this.db + .collection(MongoCollectionNames.payloadReferences) + .findOne( + { + network: this.scope.network, + genesisHash: this.scope.genesisHash, + nodeId: this.scope.nodeId, + ownerKind: 'basm-job', + ownerId: topic, + slot: previousSlot + }, + context.options() + ) + if (previous !== null) { + const previousPayload = await this.db + .collection(MongoCollectionNames.payloads) + .findOne({ _id: String(previous.payloadId) }, context.options()) + if ( + previousPayload !== null && + isPayloadKind(String(previousPayload.kind)) && + typeof previousPayload.digest === 'string' + ) { + const options = context.options() + await this.payloads.releaseReference( + options.session, + { + scope: this.scope, + payload: { + kind: previousPayload.kind as MongoPayloadKind, + digest: previousPayload.digest + }, + ownerKind: 'basm-job', + ownerId: topic, + slot: previousSlot + }, + { timeoutMS: options.timeoutMS, signal: context.signal } + ) + } + } + await this.pin(context, ref, 'basm-job', topic, this.historyUpdateSlot(nextGeneration)) } private async applyHandoff( diff --git a/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts index a34305fd2..5487e7065 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts @@ -1,8 +1,15 @@ import { createHash } from 'node:crypto' import type { Db, Document } from 'mongodb' +import { MerklePath, Transaction } from '@bsv/sdk' import type { Output } from '../../Output.js' import type { AppliedTransaction, Storage } from '../Storage.js' -import type { AdmissionPayloadRef, HistoryFence, StorageScope } from '../AdmissionStorage.js' +import { + parseStorageOutputIndex, + parseStorageUint64, + type AdmissionPayloadRef, + type HistoryFence, + type StorageScope +} from '../AdmissionStorage.js' import { MongoAdmissionStorage, type MongoAdmissionStorageOptions @@ -13,6 +20,7 @@ import { decodeMongoUint64, encodeMongoOutputIndex, encodeMongoUint64, + mongoChainKey, mongoNodeKey, mongoRecordKey } from './MongoSchema.js' @@ -127,13 +135,41 @@ export class MongoOverlayStorage implements Storage { }, { upsert: true, writeConcern: { w: 'majority', j: true } } ) + if (utxo.beef !== undefined) await this.persistTransactionBeef(utxo.txid, utxo.beef) + } + + private async persistTransactionBeef(txid: string, beef: number[]): Promise { + const tx = Transaction.fromBEEF(beef) + const published = await this.publishAdmissionPayload({ + kind: 'raw-transaction', + bytes: Buffer.from(tx.toBinary()), + txid + }) + const now = new Date() + const id = mongoRecordKey(mongoChainKey(this.admissionScope), 'transaction', txid) + await this.db.collection(MongoCollectionNames.transactions).updateOne( + { _id: id }, + { + $setOnInsert: { + _id: id, + schemaVersion: 1, + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + txid, + createdAt: now + }, + $set: { rawPayloadId: this.admission.payloadId(published), updatedAt: now } + }, + { upsert: true, writeConcern: { w: 'majority', j: true } } + ) } async findOutput( txid: string, outputIndex: number, topic?: string, - spent?: boolean + spent?: boolean, + includeBEEF = false ): Promise { const filter: Record = { network: this.admissionScope.network, @@ -152,10 +188,10 @@ export class MongoOverlayStorage implements Storage { readPreference: 'primary' }) if (document === null) return null - return this.toOutput(document) + return await this.toOutput(document, includeBEEF) } - async findOutputsForTransaction(txid: string): Promise { + async findOutputsForTransaction(txid: string, includeBEEF = false): Promise { const documents = await this.db .collection(MongoCollectionNames.outputs) .find({ @@ -165,10 +201,17 @@ export class MongoOverlayStorage implements Storage { txid }) .toArray() - return documents.map(document => this.toOutput(document)) + return await Promise.all( + documents.map(async document => await this.toOutput(document, includeBEEF)) + ) } - async findUTXOsForTopic(topic: string, since?: number, limit?: number): Promise { + async findUTXOsForTopic( + topic: string, + since?: number, + limit?: number, + includeBEEF = false + ): Promise { const filter: Record = { network: this.admissionScope.network, genesisHash: this.admissionScope.genesisHash, @@ -182,7 +225,9 @@ export class MongoOverlayStorage implements Storage { .find(filter) .sort({ score: 1, _id: 1 }) if (limit !== undefined && limit > 0) query = query.limit(limit) - return (await query.toArray()).map(document => this.toOutput(document)) + return await Promise.all( + (await query.toArray()).map(async document => await this.toOutput(document, includeBEEF)) + ) } async deleteOutput(txid: string, outputIndex: number, topic: string): Promise { @@ -335,17 +380,112 @@ export class MongoOverlayStorage implements Storage { ) } - private toOutput(document: Record): Output { - return { + private toSafeNumber(value: string, label: string): number { + const parsed = parseStorageUint64(value) + if (parsed > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new Error(`Mongo ${label} exceeds a safe JavaScript integer`) + } + return Number(parsed) + } + + private async toOutput(document: Record, includeBEEF: boolean): Promise { + const output: Output = { txid: document.txid as string, - outputIndex: Number(document.outputIndex), - outputScript: [], - satoshis: Number(decodeMongoUint64(document.satoshis as string)), + outputIndex: parseStorageOutputIndex(String(document.outputIndex)), + outputScript: await this.readScript(document), + satoshis: this.toSafeNumber(decodeMongoUint64(document.satoshis as string), 'satoshis'), topic: document.topic as string, spent: document.state === 'spent', outputsConsumed: [], consumedBy: [], - score: Number(decodeMongoUint64(document.score as string)) + score: this.toSafeNumber(decodeMongoUint64(document.score as string), 'score') + } + if (includeBEEF) { + const beef = await this.readBeef(output.txid) + if (beef !== undefined) output.beef = beef + } + return output + } + + private async readScript(document: Record): Promise { + const payloadId = document.scriptPayloadId + if (typeof payloadId !== 'string') return [] + const payload = await this.db.collection(MongoCollectionNames.payloads).findOne({ + _id: payloadId, + state: 'ready' + }) + if ( + payload === null || + typeof payload.kind !== 'string' || + typeof payload.digest !== 'string' || + !isPayloadKind(payload.kind) + ) { + throw new Error('Mongo output script payload is not ready') + } + const bytes = await this.payloads.read( + { kind: payload.kind, digest: payload.digest }, + { + offset: decodeMongoUint64(document.scriptOffset as string), + byteLength: decodeMongoUint64(document.scriptByteLength as string) + } + ) + return Array.from(bytes) + } + + private async readBeef(txid: string): Promise { + const transaction = await this.db + .collection(MongoCollectionNames.transactions) + .findOne({ + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + txid + }) + const rawPayloadId = transaction?.rawPayloadId + if (typeof rawPayloadId !== 'string') return undefined + const rawPayload = await this.db.collection(MongoCollectionNames.payloads).findOne({ + _id: rawPayloadId, + state: 'ready' + }) + if ( + rawPayload === null || + typeof rawPayload.kind !== 'string' || + typeof rawPayload.digest !== 'string' || + !isPayloadKind(rawPayload.kind) + ) { + return undefined + } + const raw = await this.payloads.read({ kind: rawPayload.kind, digest: rawPayload.digest }) + const tx = Transaction.fromBinary(Array.from(raw)) + const merkle = await this.db + .collection(MongoCollectionNames.payloadReferences) + .findOne({ + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId, + ownerKind: 'transaction', + ownerId: txid, + slot: { $regex: '^merkle-path:' } + }) + if (merkle !== null && typeof merkle.payloadId === 'string') { + const merklePayload = await this.db + .collection(MongoCollectionNames.payloads) + .findOne({ + _id: merkle.payloadId, + state: 'ready' + }) + if ( + merklePayload !== null && + typeof merklePayload.kind === 'string' && + typeof merklePayload.digest === 'string' && + isPayloadKind(merklePayload.kind) + ) { + const path = await this.payloads.read({ + kind: merklePayload.kind, + digest: merklePayload.digest + }) + tx.merklePath = MerklePath.fromBinary(Array.from(path)) + } } + return tx.merklePath === undefined ? tx.toBEEF() : tx.toAtomicBEEF() } } diff --git a/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts b/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts index ab899ecd2..2077d0f6d 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts @@ -186,6 +186,29 @@ export class MongoPayloadStore { }) } + public async read( + payload: Pick, + range?: { offset: string; byteLength: string } + ): Promise { + const payloadId = this.payloadId(this.scope, payload) + const record = await this.db + .collection(MongoCollectionNames.payloads) + .findOne({ + _id: payloadId, + state: 'ready' + }) + if (record === null) throw new Error('Mongo payload is not ready') + const total = BigInt(decodeMongoUint64(record.byteLength)) + const offset = range === undefined ? BigInt(0) : BigInt(range.offset) + const length = range === undefined ? total : BigInt(range.byteLength) + if (offset < BigInt(0) || length < BigInt(0) || offset + length > total) + throw new Error('Mongo payload range is invalid') + if (offset + length > BigInt(Number.MAX_SAFE_INTEGER)) + throw new Error('Mongo payload range exceeds safe integer') + const bytes = await this.readBytes(record) + return bytes.subarray(Number(offset), Number(offset + length)) + } + public async publish(input: MongoPayloadInput): Promise { input = { ...input } this.validateInput(input) @@ -746,6 +769,28 @@ export class MongoPayloadStore { if (file !== null) await this.bucket.delete(fileId).catch(() => undefined) } + private async readBytes(record: PayloadDocument): Promise { + if (record.inlineData !== undefined) return Buffer.from(record.inlineData.buffer) + if (record.fileId === undefined) throw new Error('Mongo ready payload has no bytes') + const chunks: Buffer[] = [] + const download = this.bucket.openDownloadStream(record.fileId) + const iterator = download[Symbol.asyncIterator]() + try { + while (true) { + const next = await iterator.next() + if (next.done) break + const chunk = next.value + if (!(chunk instanceof Uint8Array)) + throw new Error('Mongo GridFS returned a non-byte chunk') + chunks.push(Buffer.from(chunk)) + } + } catch (error) { + download.destroy(error instanceof Error ? error : new Error('Mongo payload read failed')) + throw error + } + return Buffer.concat(chunks) + } + private async verifyGridFs( fileId: ObjectId, input: Pick, From f8a6cac048104814eb8d9b1523dfacf0f29503fc Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:00:08 -0500 Subject: [PATCH 016/168] fix(sonar): overlay S03 admission complexity and empty method Extract Engine submit and admission plan validators, use localeCompare and structuredClone, and throw from unimplemented Mongo BEEF updates. --- packages/overlays/overlay/src/Engine.ts | 133 +++++++---- .../overlays/overlay/src/EngineAdmission.ts | 3 +- .../__tests/mongo/MongoAdmissionHarness.ts | 2 +- .../src/storage/mongo/MongoAdmissionPlan.ts | 209 +++++++++--------- .../src/storage/mongo/MongoOverlayStorage.ts | 4 +- 5 files changed, 203 insertions(+), 148 deletions(-) diff --git a/packages/overlays/overlay/src/Engine.ts b/packages/overlays/overlay/src/Engine.ts index 0e8a87411..29e5d81c5 100644 --- a/packages/overlays/overlay/src/Engine.ts +++ b/packages/overlays/overlay/src/Engine.ts @@ -110,6 +110,19 @@ type StorageMutationContext = { offChainValues: number[] | undefined } +type OverlayAdmissionSubmitContext = { + taggedBEEF: TaggedBEEF + steak: STEAK + tx: Transaction + txid: string + mode: SubmissionMode + offChainValues: number[] | undefined + validations: TopicValidation[] + failedTopics: Set + anyTopicAccepted: boolean + onSteakReady?: (steak: STEAK) => void +} + function findSpendingInputIndex(tx: Transaction, output: Output): number { return tx.inputs.findIndex(input => { const realSource = input.sourceTXID || input.sourceTransaction?.id('hex') @@ -1043,6 +1056,69 @@ export class Engine { this.endTime(`transactionPropagation_${txid.substring(0, 10)}`) } + private assertSupportedTopics(topics: string[]): void { + for (const topic of topics) { + if (this.managers[topic] === undefined || this.managers[topic] === null) { + throw new Error(`This server does not support this topic: ${topic}`) + } + } + } + + private shouldSkipPropagation(mode: SubmissionMode): boolean { + return ( + this.advertiser === undefined || + mode === 'historical-tx' || + mode === 'historical-tx-no-spv' + ) + } + + private async acknowledgeOverlayAdmission( + context: OverlayAdmissionSubmitContext + ): Promise { + const admissionHost = getOverlayAdmissionHost(this.storage) + if (admissionHost === undefined) return undefined + const { + taggedBEEF, + steak, + tx, + txid, + mode, + offChainValues, + validations, + failedTopics, + anyTopicAccepted, + onSteakReady + } = context + if (!anyTopicAccepted) { + if (onSteakReady !== undefined) onSteakReady(steak) + return steak + } + const applied = await this.buildAppliedTransactionRecord(tx) + const buildPlan = async () => + await buildOverlayAdmissionPlan({ + host: admissionHost, + tx, + txid, + beef: taggedBEEF.beef, + topics: taggedBEEF.topics, + mode: overlayAdmissionMode(mode), + offChainValues, + validations, + failedTopics, + lookupServices: this.lookupServices, + includePropagation: !this.shouldSkipPropagation(mode), + applied + }) + const committed = await waitForAdmissionReceipt( + admissionHost.admission, + await buildPlan(), + buildPlan + ) + const acknowledged = JSON.parse(committed.receipt.steak) as STEAK + if (onSteakReady !== undefined) onSteakReady(acknowledged) + return acknowledged + } + /** * Submits a transaction for processing by Overlay Services. * @param {TaggedBEEF} taggedBEEF - The transaction to process @@ -1055,11 +1131,7 @@ export class Engine { * @returns {Promise} The submitted transaction execution acknowledgement */ async submit(taggedBEEF: TaggedBEEF, onSteakReady?: (steak: STEAK) => void, mode: 'historical-tx' | 'current-tx' | 'historical-tx-no-spv' = 'current-tx', offChainValues?: number[]): Promise { - for (const t of taggedBEEF.topics) { - if (this.managers[t] === undefined || this.managers[t] === null) { - throw new Error(`This server does not support this topic: ${t}`) - } - } + this.assertSupportedTopics(taggedBEEF.topics) // Validate the transaction SPV information const tx = Transaction.fromBEEF(taggedBEEF.beef) @@ -1122,42 +1194,19 @@ export class Engine { throw error } - const admissionHost = getOverlayAdmissionHost(this.storage) - if (admissionHost !== undefined) { - if (!anyTopicAccepted) { - if (onSteakReady !== undefined) onSteakReady(steak) - return steak - } - const admissionMode = overlayAdmissionMode(mode) - const includePropagation = - this.advertiser !== undefined && - mode !== 'historical-tx' && - mode !== 'historical-tx-no-spv' - const applied = await this.buildAppliedTransactionRecord(tx) - const buildPlan = async () => - await buildOverlayAdmissionPlan({ - host: admissionHost, - tx, - txid, - beef: taggedBEEF.beef, - topics: taggedBEEF.topics, - mode: admissionMode, - offChainValues, - validations, - failedTopics, - lookupServices: this.lookupServices, - includePropagation, - applied - }) - const committed = await waitForAdmissionReceipt( - admissionHost.admission, - await buildPlan(), - buildPlan - ) - const acknowledged = JSON.parse(committed.receipt.steak) as STEAK - if (onSteakReady !== undefined) onSteakReady(acknowledged) - return acknowledged - } + const admissionSteak = await this.acknowledgeOverlayAdmission({ + taggedBEEF, + steak, + tx, + txid, + mode, + offChainValues, + validations, + failedTopics, + anyTopicAccepted, + onSteakReady + }) + if (admissionSteak !== undefined) return admissionSteak // Call the callback function with STEAK if it is provided (before storage mutations) if (onSteakReady !== undefined) { @@ -1187,7 +1236,7 @@ export class Engine { }) // If we don't have an advertiser or we are dealing with historical transactions, just return the steak - if (this.advertiser === undefined || mode === 'historical-tx' || mode === 'historical-tx-no-spv') { + if (this.shouldSkipPropagation(mode)) { return steak } diff --git a/packages/overlays/overlay/src/EngineAdmission.ts b/packages/overlays/overlay/src/EngineAdmission.ts index 0f2e81fab..19e451cc9 100644 --- a/packages/overlays/overlay/src/EngineAdmission.ts +++ b/packages/overlays/overlay/src/EngineAdmission.ts @@ -80,7 +80,8 @@ export function overlayAdmissionOperationId( txid: string, topics: string[] ): string { - const raw = `submit:${mode}:${txid}:${[...topics].sort().join('\n')}` + const topicKey = [...topics].sort((a, b) => a.localeCompare(b, 'en')).join('\n') + const raw = `submit:${mode}:${txid}:${topicKey}` if (Buffer.byteLength(raw, 'utf8') <= 512 && raw.isWellFormed()) return raw return `submit:${createHash('sha256').update(raw, 'utf8').digest('hex')}` } diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts index 0e92737b3..953ae241f 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionHarness.ts @@ -30,7 +30,7 @@ import { import type { MongoReplicaFixture } from './MongoReplicaFixture.js' import { referenceScope } from '../admission/ReferenceAdmissionStorage.js' -const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T +const clone = (value: T): T => structuredClone(value) const scopeKey = (scope: StorageScope): string => JSON.stringify([scope.network, scope.genesisHash, scope.nodeId]) const operationKey = (key: AdmissionOperationKey): string => diff --git a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts index 43769e50a..df3bc8f01 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionPlan.ts @@ -6,8 +6,10 @@ import { type AdmissionCommitResult, type AdmissionOutboxIntent, type AdmissionOutpoint, + type AdmissionOutput, type AdmissionPayloadRef, type AdmissionReceipt, + type AdmissionTopicDecision, type StorageScope } from '../AdmissionStorage.js' @@ -57,10 +59,9 @@ export const samePayload = ( left: AdmissionPayloadRef | undefined, right: AdmissionPayloadRef ): boolean => - left !== undefined && - left.digest === right.digest && - left.byteLength === right.byteLength && - left.kind === right.kind + left?.digest === right.digest && + left?.byteLength === right.byteLength && + left?.kind === right.kind export const admissionPlanPayloads = (plan: AdmissionCommit): AdmissionPayloadRef[] => { const references = [...plan.payloads] @@ -109,9 +110,23 @@ export const isBoundSteak = (plan: AdmissionCommit): boolean => { }) } -export const validateAdmissionPlan = ( - plan: AdmissionCommit -): AdmissionRejectionCode | undefined => { +const hasInvalidTopicSet = (plan: AdmissionCommit): boolean => { + const topics = new Set(plan.identity.topics.map(item => item.topic)) + const decisionTopics = new Set(plan.decisions.map(decision => decision.topic)) + return ( + topics.size !== plan.identity.topics.length || + plan.decisions.length !== topics.size || + decisionTopics.size !== plan.decisions.length || + decisionTopics.size !== topics.size || + plan.decisions.some(decision => !topics.has(decision.topic)) || + new Set(plan.outbox.map(intent => intent.eventId)).size !== plan.outbox.length + ) +} + +const hasHistoricalPropagation = (plan: AdmissionCommit): boolean => + plan.identity.mode === 'historical' && plan.outbox.some(intent => intent.kind === 'propagation') + +const identityRejection = (plan: AdmissionCommit): AdmissionRejectionCode | undefined => { let semanticDigest: string try { semanticDigest = admissionSemanticDigest(plan.identity) @@ -124,112 +139,100 @@ export const validateAdmissionPlan = ( ) { return 'digest-mismatch' } - const topics = new Set(plan.identity.topics.map(item => item.topic)) - const decisionTopics = new Set(plan.decisions.map(decision => decision.topic)) + if (hasInvalidTopicSet(plan) || hasHistoricalPropagation(plan)) return 'invalid-plan' + return undefined +} + +const hasUnreadyPayloads = (plan: AdmissionCommit): boolean => + admissionPlanPayloads(plan).some( + ref => !isHash(ref.digest) || !isUint64(ref.byteLength) || ref.digest.length === 0 + ) + +const isInvalidOutboxIntent = (intent: AdmissionOutboxIntent): boolean => + intent.eventId.length === 0 || + !intent.eventId.isWellFormed() || + intent.target.length === 0 || + !intent.target.isWellFormed() || + (intent.kind !== 'lookup' && intent.kind !== 'propagation') + +const isInvalidSpend = ( + spend: { outpoint: AdmissionOutpoint; spender: string }, + txid: string +): boolean => !isWireOutpoint(spend.outpoint) || spend.spender !== txid + +const isInvalidEdge = (edge: { + source: AdmissionOutpoint + consumer: AdmissionOutpoint +}): boolean => !isWireOutpoint(edge.source) || !isWireOutpoint(edge.consumer) + +const isInvalidOutput = (output: AdmissionOutput, txid: string): boolean => { + if (output.txid !== txid || !isWireOutpoint(output)) return true if ( - topics.size !== plan.identity.topics.length || - plan.decisions.length !== topics.size || - decisionTopics.size !== plan.decisions.length || - decisionTopics.size !== topics.size || - plan.decisions.some(decision => !topics.has(decision.topic)) || - new Set(plan.outbox.map(intent => intent.eventId)).size !== plan.outbox.length + ![output.satoshis, output.score, output.script.offset, output.script.byteLength].every(isUint64) ) { - return 'invalid-plan' + return true + } + if (!isHash(output.script.payload.digest) || !isUint64(output.script.payload.byteLength)) { + return true } + return ( + parseStorageUint64(output.script.offset) + parseStorageUint64(output.script.byteLength) > + parseStorageUint64(output.script.payload.byteLength) + ) +} + +const isInvalidApplied = (applied: AdmissionTopicDecision['applied'], txid: string): boolean => { + if (applied.txid !== txid || !isHash(applied.txid)) return true + if (applied.firstSeenHeight !== undefined && !isUint64(applied.firstSeenHeight)) return true if ( - plan.identity.mode === 'historical' && - plan.outbox.some(intent => intent.kind === 'propagation') + applied.proof !== undefined && + (!isHash(applied.proof.digest) || !isUint64(applied.proof.byteLength)) ) { - return 'invalid-plan' + return true } - const payloads = admissionPlanPayloads(plan) + return ( + applied.block !== undefined && + (![applied.block.height, applied.block.index].every(isUint64) || + !isHash(applied.block.hash) || + !isHash(applied.block.merkleRoot)) + ) +} + +const isInvalidHistoryUpdate = (decision: AdmissionTopicDecision): boolean => { + const update = decision.historyUpdate + if (update === undefined) return false + return ( + !isUint64(update.nextTopicHistoryGeneration) || + !isUint64(update.affectedFromHeight) || + parseStorageUint64(update.nextTopicHistoryGeneration) <= + parseStorageUint64(decision.expectedHistory.topicHistoryGeneration) + ) +} + +const isInvalidDecision = (decision: AdmissionTopicDecision, txid: string): boolean => { if ( - payloads.some( - ref => !isHash(ref.digest) || !isUint64(ref.byteLength) || ref.digest.length === 0 - ) + !isUint64(decision.expectedHistory.chainEpoch) || + !isUint64(decision.expectedHistory.topicHistoryGeneration) ) { - return 'payload-not-ready' + return true } + if (decision.spends.some(spend => isInvalidSpend(spend, txid))) return true + if (decision.evictions.some(eviction => !isWireOutpoint(eviction))) return true + if (decision.outputs.some(output => isInvalidOutput(output, txid))) return true + if (decision.edges.some(isInvalidEdge)) return true + return isInvalidApplied(decision.applied, txid) || isInvalidHistoryUpdate(decision) +} + +export const validateAdmissionPlan = ( + plan: AdmissionCommit +): AdmissionRejectionCode | undefined => { + const rejected = identityRejection(plan) + if (rejected !== undefined) return rejected + if (hasUnreadyPayloads(plan)) return 'payload-not-ready' if (!isBoundSteak(plan)) return 'invalid-plan' - for (const intent of plan.outbox) { - if ( - intent.eventId.length === 0 || - !intent.eventId.isWellFormed() || - intent.target.length === 0 || - !intent.target.isWellFormed() || - (intent.kind !== 'lookup' && intent.kind !== 'propagation') - ) { - return 'invalid-plan' - } - } - for (const decision of plan.decisions) { - if ( - !isUint64(decision.expectedHistory.chainEpoch) || - !isUint64(decision.expectedHistory.topicHistoryGeneration) - ) { - return 'invalid-plan' - } - if ( - decision.spends.some( - spend => !isWireOutpoint(spend.outpoint) || spend.spender !== plan.identity.txid - ) - ) { - return 'invalid-plan' - } - if (decision.evictions.some(eviction => !isWireOutpoint(eviction))) return 'invalid-plan' - if ( - decision.outputs.some(output => { - if (output.txid !== plan.identity.txid || !isWireOutpoint(output)) return true - if ( - ![output.satoshis, output.score, output.script.offset, output.script.byteLength].every( - isUint64 - ) - ) { - return true - } - if (!isHash(output.script.payload.digest) || !isUint64(output.script.payload.byteLength)) - return true - return ( - parseStorageUint64(output.script.offset) + parseStorageUint64(output.script.byteLength) > - parseStorageUint64(output.script.payload.byteLength) - ) - }) - ) { - return 'invalid-plan' - } - if ( - decision.edges.some(edge => !isWireOutpoint(edge.source) || !isWireOutpoint(edge.consumer)) - ) { - return 'invalid-plan' - } - const applied = decision.applied - if (applied.txid !== plan.identity.txid || !isHash(applied.txid)) return 'invalid-plan' - if (applied.firstSeenHeight !== undefined && !isUint64(applied.firstSeenHeight)) - return 'invalid-plan' - if ( - applied.proof !== undefined && - (!isHash(applied.proof.digest) || !isUint64(applied.proof.byteLength)) - ) { - return 'invalid-plan' - } - if ( - applied.block !== undefined && - (![applied.block.height, applied.block.index].every(isUint64) || - !isHash(applied.block.hash) || - !isHash(applied.block.merkleRoot)) - ) { - return 'invalid-plan' - } - if (decision.historyUpdate !== undefined) { - if ( - !isUint64(decision.historyUpdate.nextTopicHistoryGeneration) || - !isUint64(decision.historyUpdate.affectedFromHeight) || - parseStorageUint64(decision.historyUpdate.nextTopicHistoryGeneration) <= - parseStorageUint64(decision.expectedHistory.topicHistoryGeneration) - ) { - return 'invalid-plan' - } - } + if (plan.outbox.some(isInvalidOutboxIntent)) return 'invalid-plan' + if (plan.decisions.some(decision => isInvalidDecision(decision, plan.identity.txid))) { + return 'invalid-plan' } return undefined } diff --git a/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts index 5487e7065..1e303f951 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts @@ -290,7 +290,9 @@ export class MongoOverlayStorage implements Storage { } } - async updateTransactionBEEF(_txid: string, _beef: number[]): Promise {} + async updateTransactionBEEF(txid: string, _beef: number[]): Promise { + throw new Error(`Mongo overlay storage does not implement updateTransactionBEEF for ${txid}`) + } async insertAppliedTransaction(tx: AppliedTransaction): Promise { const now = new Date() From 134d6fb77cadbf07b2fbf2dca0dc12ba973b5c86 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:00:47 -0500 Subject: [PATCH 017/168] fix(sonar): evidence coordinator and chaintracks findings Drop redundant optional-undefined on lookup evidence limits, extract parseEvidence and Chaintracks header-retry helpers below S3776, and rename coordinator catch params to error_. --- .../sdk/src/overlay-tools/LookupResolver.ts | 2 +- .../src/transaction/TransactionEvidence.ts | 80 ++++++++--- .../TransactionEvidenceCoordinator.ts | 16 +-- .../chaintracker/ChaintracksChainTracker.ts | 128 ++++++++++-------- 4 files changed, 140 insertions(+), 86 deletions(-) diff --git a/packages/sdk/src/overlay-tools/LookupResolver.ts b/packages/sdk/src/overlay-tools/LookupResolver.ts index eac559734..640b5cfae 100644 --- a/packages/sdk/src/overlay-tools/LookupResolver.ts +++ b/packages/sdk/src/overlay-tools/LookupResolver.ts @@ -524,7 +524,7 @@ interface LookupQuerySessionOptions { softTimeoutMs?: number waitForAllHosts: boolean correlationId?: string - evidenceLimits?: LookupQueryOptions['evidenceLimits'] + evidenceLimits?: { maxOutputs?: number; maxBytes?: number } resolveTxId: (output: LookupAnswer['outputs'][number], now: number) => string | null } diff --git a/packages/sdk/src/transaction/TransactionEvidence.ts b/packages/sdk/src/transaction/TransactionEvidence.ts index 070bbea6c..89b501a4c 100644 --- a/packages/sdk/src/transaction/TransactionEvidence.ts +++ b/packages/sdk/src/transaction/TransactionEvidence.ts @@ -86,32 +86,73 @@ export interface EvidenceCandidate { graphBinding: string } -export function parseEvidence( +function isInvalidEvidenceByte(byte: number): boolean { + return !Number.isInteger(byte) || byte < 0 || byte > 255 +} + +function snapshotEvidenceBytes( evidence: TransactionEvidence, limits: TransactionEvidenceLimits -): EvidenceCandidate { +): number[] { const outputIndex = evidence.outputIndex - const hint = evidence.txid if (!Array.isArray(evidence.beef) || !Number.isSafeInteger(outputIndex) || outputIndex < 0) { throw new TransactionEvidenceError('invalid-evidence') } if (evidence.beef.length > limits.candidateBytes) throw new TransactionEvidenceError('limit') // Snapshot and validate BEFORE either parsing or computing the receipt digest. const bytes = evidence.beef.slice() - if (bytes.length === 0 || bytes.some(byte => !Number.isInteger(byte) || byte < 0 || byte > 255)) { + if (bytes.length === 0 || bytes.some(isInvalidEvidenceByte)) { throw new TransactionEvidenceError('invalid-evidence') } + return bytes +} + +function loadEvidenceTransaction( + bytes: number[], + limits: TransactionEvidenceLimits, + hint: string | undefined, + outputIndex: number +): Transaction { const beef = Beef.fromBinary(bytes) if (beef.txs.length > limits.transactions) throw new TransactionEvidenceError('limit') const target = beef.atomicTxid ?? beef.txs.at(-1)?.txid - const tx = target === undefined ? undefined : beef.findAtomicTransaction(target) + if (target === undefined) throw new TransactionEvidenceError('invalid-evidence') + const tx = beef.findAtomicTransaction(target) if (tx === undefined) throw new TransactionEvidenceError('invalid-evidence') const txid = tx.id('hex') if (hint !== undefined && (typeof hint !== 'string' || hint.toLowerCase() !== txid)) { throw new TransactionEvidenceError('invalid-evidence') } if (outputIndex >= tx.outputs.length) throw new TransactionEvidenceError('invalid-evidence') + return tx +} + +function graphScriptBytes(tx: Transaction): number { + let scriptBytes = 0 + for (const output of tx.outputs) scriptBytes += output.lockingScript.toBinary().length + for (const input of tx.inputs) scriptBytes += input.unlockingScript?.toBinary().length ?? 0 + return scriptBytes +} + +function enqueueUnconfirmedParents( + current: Transaction, + pending: Transaction[], + spent: Set +): void { + for (const input of current.inputs) { + if (input.sourceTransaction == null) throw new TransactionEvidenceError('invalid-evidence') + const sourceTxid = input.sourceTransaction.id('hex') + if (input.sourceTXID !== undefined && input.sourceTXID !== sourceTxid) { + throw new TransactionEvidenceError('invalid-evidence') + } + const outpoint = `${sourceTxid}.${input.sourceOutputIndex}` + if (spent.has(outpoint)) throw new TransactionEvidenceError('invalid-evidence') + spent.add(outpoint) + pending.push(input.sourceTransaction) + } +} +function assertUnconfirmedGraph(tx: Transaction, limits: TransactionEvidenceLimits): void { // Always walk the COMPLETE unconfirmed graph, including on a positive cache hit. // A per-ancestor verdict cannot establish graph-wide spend consistency. const pending = [tx] @@ -124,27 +165,28 @@ export function parseEvidence( const id = current.id('hex') if (visited.has(id)) continue visited.add(id) - for (const output of current.outputs) scriptBytes += output.lockingScript.toBinary().length - for (const input of current.inputs) scriptBytes += input.unlockingScript?.toBinary().length ?? 0 + scriptBytes += graphScriptBytes(current) inputs += current.inputs.length - if (inputs > limits.inputs || scriptBytes > limits.scriptBytes) + if (inputs > limits.inputs || scriptBytes > limits.scriptBytes) { throw new TransactionEvidenceError('limit') + } if (current.merklePath != null) continue if (current.inputs.length === 0) throw new TransactionEvidenceError('invalid-evidence') - for (const input of current.inputs) { - if (input.sourceTransaction == null) throw new TransactionEvidenceError('invalid-evidence') - const sourceTxid = input.sourceTransaction.id('hex') - if (input.sourceTXID !== undefined && input.sourceTXID !== sourceTxid) - throw new TransactionEvidenceError('invalid-evidence') - const outpoint = `${sourceTxid}.${input.sourceOutputIndex}` - if (spent.has(outpoint)) throw new TransactionEvidenceError('invalid-evidence') - spent.add(outpoint) - pending.push(input.sourceTransaction) - } + enqueueUnconfirmedParents(current, pending, spent) } +} + +export function parseEvidence( + evidence: TransactionEvidence, + limits: TransactionEvidenceLimits +): EvidenceCandidate { + const outputIndex = evidence.outputIndex + const bytes = snapshotEvidenceBytes(evidence, limits) + const tx = loadEvidenceTransaction(bytes, limits, evidence.txid, outputIndex) + assertUnconfirmedGraph(tx, limits) return { tx, - txid, + txid: tx.id('hex'), receipt: toHex(sha256(bytes)), byteLength: bytes.length, outputIndex, diff --git a/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts b/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts index 921c67158..bdeb252c6 100644 --- a/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts +++ b/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts @@ -173,8 +173,8 @@ export class TransactionEvidenceCoordinator { let candidate: EvidenceCandidate try { candidate = parseEvidence(evidence, this.limits) - } catch (error) { - throw outcome(error) + } catch (error_) { + throw outcome(error_) } let job = this.work.get(candidate.txid) if (job === undefined) { @@ -287,8 +287,8 @@ export class TransactionEvidenceCoordinator { this.check(job, job.controller.signal) this.finish(job, cached) return - } catch (failure) { - error = outcome(failure) + } catch (error_) { + error = outcome(error_) this.removePositive(job.txid) } } @@ -363,12 +363,12 @@ export class TransactionEvidenceCoordinator { this.cache(job.txid, positive) this.finish(job, positive) return - } catch (failure) { - error = outcome(failure) + } catch (error_) { + error = outcome(error_) } } - } catch (failure) { - error = outcome(failure) + } catch (error_) { + error = outcome(error_) } this.finish(job, undefined, error) } diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/ChaintracksChainTracker.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/ChaintracksChainTracker.ts index 0a90d1ad8..fdbfcc9e3 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/ChaintracksChainTracker.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/ChaintracksChainTracker.ts @@ -179,68 +179,89 @@ export class ChaintracksChainTracker implements ChainTracker { parent?: TelemetrySpan, signal?: AbortSignal ): Promise { - const chaintracks = this.chaintracks - const context = this.verificationContext - let header: BlockHeader | undefined - - const retries = Math.max(1, this.options.maxRetries ?? 6) - const retryDelayMs = this.options.retryDelayMs ?? 250 + const header = await this.fetchCanonicalHeader( + this.chaintracks, + this.verificationContext, + height, + parent, + signal + ) + if (header == null) return false - let error: WalletError | undefined + // Diagnostic only: a root is always freshly read from the current canonical source above. + this.cache[height] = header.merkleRoot + this.cacheUpdatedAt[height] = Date.now() + this.pruneDiagnosticCache() - for (let tryCount = 1; tryCount <= retries; tryCount++) { - try { - this.throwIfAborted(signal) - if (this.chaintracks !== chaintracks || this.verificationContext !== context) { - throw new Error('Chaintracks provider changed during header lookup') - } - header = - parent == null - ? await chaintracks.findHeaderForHeight(height) - : await this.telemetry.withSpan( - 'wallet.chaintracks.find_header', - { - component: 'chaintracks-chain-tracker', - kind: 'client', - parent: parent.context, - attributes: { - 'retry.attempt': tryCount - } - }, - async () => await chaintracks.findHeaderForHeight(height) - ) + const valid = header.merkleRoot === root + parent?.end({ + attributes: { + 'chaintracks.cache_hit': false, + 'chaintracks.valid': valid + } + }) + return valid + } - if (this.chaintracks !== chaintracks || this.verificationContext !== context) { - throw new Error('Chaintracks provider changed during header lookup') - } + private assertProviderUnchanged(chaintracks: ChaintracksClientApi, context: number): void { + if (this.chaintracks !== chaintracks || this.verificationContext !== context) { + throw new Error('Chaintracks provider changed during header lookup') + } + } - if (header == null) { - if (tryCount >= retries) return false - this.throwIfAborted(signal) - await wait(retryDelayMs) - continue + private async readHeaderForHeight( + chaintracks: ChaintracksClientApi, + height: number, + tryCount: number, + parent?: TelemetrySpan + ): Promise { + if (parent == null) return await chaintracks.findHeaderForHeight(height) + return await this.telemetry.withSpan( + 'wallet.chaintracks.find_header', + { + component: 'chaintracks-chain-tracker', + kind: 'client', + parent: parent.context, + attributes: { + 'retry.attempt': tryCount } + }, + async () => await chaintracks.findHeaderForHeight(height) + ) + } - break + private async fetchCanonicalHeader( + chaintracks: ChaintracksClientApi, + context: number, + height: number, + parent: TelemetrySpan | undefined, + signal: AbortSignal | undefined + ): Promise { + const retries = Math.max(1, this.options.maxRetries ?? 6) + const retryDelayMs = this.options.retryDelayMs ?? 250 + for (let tryCount = 1; tryCount <= retries; tryCount++) { + try { + this.throwIfAborted(signal) + this.assertProviderUnchanged(chaintracks, context) + const header = await this.readHeaderForHeight(chaintracks, height, tryCount, parent) + this.assertProviderUnchanged(chaintracks, context) + if (header != null) return header + if (tryCount >= retries) return undefined + this.throwIfAborted(signal) + await wait(retryDelayMs) } catch (error_: unknown) { this.throwIfAborted(signal) - if (this.chaintracks !== chaintracks || this.verificationContext !== context) { - throw new Error('Chaintracks provider changed during header lookup') - } - error = WalletError.fromUnknown(error_) - if (tryCount >= retries) { - throw error - } + this.assertProviderUnchanged(chaintracks, context) + const error = WalletError.fromUnknown(error_) + if (tryCount >= retries) throw error this.throwIfAborted(signal) await wait(retryDelayMs) } } + return undefined + } - if (header == null) return false - - // Diagnostic only: a root is always freshly read from the current canonical source above. - this.cache[height] = header.merkleRoot - this.cacheUpdatedAt[height] = Date.now() + private pruneDiagnosticCache(): void { const now = Date.now() const entries = Object.entries(this.cacheUpdatedAt).sort(([, a], [, b]) => a - b) for (const [index, [cachedHeight, updatedAt]] of entries.entries()) { @@ -248,15 +269,6 @@ export class ChaintracksChainTracker implements ChainTracker { delete this.cache[Number(cachedHeight)] delete this.cacheUpdatedAt[Number(cachedHeight)] } - - const valid = header.merkleRoot === root - parent?.end({ - attributes: { - 'chaintracks.cache_hit': false, - 'chaintracks.valid': valid - } - }) - return valid } private throwIfAborted(signal?: AbortSignal): void { From e419d8e0e183cf33d49f458ab7ea3ad1481a20c4 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:01:21 -0500 Subject: [PATCH 018/168] fix(sonar): overlay BASM protocol findings Extract BASM reconcile helpers so cognitive complexity stays at or below 15, prefer optional chaining on nullable anchors, and invert the advertised response-length comparison. --- packages/overlays/overlay/src/BASMRemote.ts | 2 +- packages/overlays/overlay/src/Engine.ts | 175 +++++++++++++------- 2 files changed, 117 insertions(+), 60 deletions(-) diff --git a/packages/overlays/overlay/src/BASMRemote.ts b/packages/overlays/overlay/src/BASMRemote.ts index d426520ca..bcb09ffd9 100644 --- a/packages/overlays/overlay/src/BASMRemote.ts +++ b/packages/overlays/overlay/src/BASMRemote.ts @@ -168,7 +168,7 @@ export class BASMRemote { private async readResponse(response: Response, signal: AbortSignal): Promise { const advertisedLength = response.headers?.get('content-length') if (advertisedLength !== null && advertisedLength !== undefined) { - if (!(Number(advertisedLength) <= this.limits.maxResponseBytes)) { + if (Number(advertisedLength) > this.limits.maxResponseBytes) { void response.body?.cancel().catch(() => {}) requireBASMLimit(false, 'BASM response exceeds byte limit') } diff --git a/packages/overlays/overlay/src/Engine.ts b/packages/overlays/overlay/src/Engine.ts index 6e88455ed..e0faff35a 100644 --- a/packages/overlays/overlay/src/Engine.ts +++ b/packages/overlays/overlay/src/Engine.ts @@ -116,6 +116,12 @@ function findSpendingInputIndex(tx: Transaction, output: Output): number { }) } +function requireBASMDefined(value: T | undefined, aligned: boolean, message: string): T { + requireBASM(aligned, message) + requireBASM(value !== undefined, message) + return value +} + /** * An engine for running BSV Overlay Services (topic managers and lookup services). */ @@ -1685,70 +1691,120 @@ export class Engine { ]) report.localTip = localTip report.remoteTip = remoteTip - - if (remoteTip.blockHeight >= 0) { - const tipRange = await remote.requestTopicAnchorRange(remoteTip.blockHeight, remoteTip.blockHeight) - const tipAnchor = tipRange.anchors[0] - requireBASM(tipAnchor !== undefined && tipAnchor.tac === remoteTip.tac, 'BASM tip does not match its anchor') - for (const field of ['blockHash', 'basmRoot', 'admittedCount'] as const) { - requireBASM(remoteTip[field] === undefined || remoteTip[field] === tipAnchor[field], 'BASM tip metadata does not match its anchor') - } - await this.requireCanonicalBASMAnchor(tipAnchor) - } - + await this.requireMatchingRemoteBASMTip(remote, remoteTip) if (localTip.blockHeight >= remoteTip.blockHeight) { - if (localTip.blockHeight >= 0 && localTip.blockHeight === remoteTip.blockHeight && localTip.tac === remoteTip.tac) { - const localAnchor = await this.storage.findTopicBlockAnchor?.(topic, localTip.blockHeight) - requireBASM(localAnchor !== undefined && localAnchor.tac === localTip.tac, 'Local BASM tip lacks its anchor') - await this.requireCanonicalBASMAnchor(localAnchor) - } - report.status = localTip.tac === remoteTip.tac && localTip.blockHeight === remoteTip.blockHeight ? 'matched' : 'diverged' - report.message = report.status === 'matched' - ? 'Topic anchor tips match' - : 'Remote tip is not ahead; historical divergence needs manual or binary-search reconciliation' - return report - } - - const fromHeight = localTip.blockHeight < 0 - ? Math.max(remoteTip.blockHeight - DEFAULT_BASM_SYNC_PAGE_SIZE + 1, 0) - : localTip.blockHeight + 1 - const toHeight = Math.min(fromHeight + DEFAULT_BASM_SYNC_PAGE_SIZE - 1, remoteTip.blockHeight) - const range = await remote.requestTopicAnchorRange(fromHeight, toHeight) - requireBASM(range.anchors.length > 0 && range.anchors.at(-1)?.blockHeight === toHeight, 'BASM range omits its requested target') - requireBASM(localTip.blockHeight < 0 || range.anchors[0].blockHeight === fromHeight, 'BASM range omits its next height') - let previousTac = localTip.tac - for (const anchor of range.anchors) { - requireBASM(anchor.tac === computeTac(previousTac, anchor.blockHash, anchor.basmRoot), 'BASM range TAC is inconsistent with its prefix') - previousTac = anchor.tac - } - if (toHeight === remoteTip.blockHeight) requireBASM(previousTac === remoteTip.tac, 'BASM range differs from its tip') - for (const remoteAnchor of range.anchors) { - await this.reconcileRemoteAnchor(topic, remote, remoteAnchor, report) - if (report.status === 'diverged') return report - } - - const finalRemoteTip = await remote.requestTopicAnchorTip() - requireBASM(finalRemoteTip.blockHeight === remoteTip.blockHeight && finalRemoteTip.tac === remoteTip.tac, 'BASM peer history changed during reconciliation') - - const refreshedTip = await this.provideTopicAnchorTip(topic) - report.localTip = refreshedTip - report.status = refreshedTip.blockHeight === remoteTip.blockHeight && refreshedTip.tac === remoteTip.tac ? 'matched' : 'advanced' - return report + return await this.finishBASMWhenRemoteIsNotAhead(topic, localTip, remoteTip, report) + } + return await this.advanceBASMWithRemoteAnchors(topic, remote, localTip, remoteTip, report) } catch (error) { - report.status = 'error' - if (error instanceof Error && 'code' in error && typeof error.code === 'string') report.errorCode = error.code - report.message = error instanceof Error ? error.message : String(error) - this.logger.error(`[BASM SYNC] Sync failed for topic "${topic}" with peer "${endpoint}"`, error) - return report + return this.markBASMPeerSyncError(report, topic, endpoint, error) } } + private async requireMatchingRemoteBASMTip(remote: BASMRemote, remoteTip: TopicAnchorTip): Promise { + if (remoteTip.blockHeight < 0) { + return + } + const tipRange = await remote.requestTopicAnchorRange(remoteTip.blockHeight, remoteTip.blockHeight) + const remoteTipAnchor = tipRange.anchors[0] + const tipAnchor = requireBASMDefined( + remoteTipAnchor, + remoteTipAnchor?.tac === remoteTip.tac, + 'BASM tip does not match its anchor' + ) + for (const field of ['blockHash', 'basmRoot', 'admittedCount'] as const) { + requireBASM(remoteTip[field] === undefined || remoteTip[field] === tipAnchor[field], 'BASM tip metadata does not match its anchor') + } + await this.requireCanonicalBASMAnchor(tipAnchor) + } + + private async finishBASMWhenRemoteIsNotAhead( + topic: string, + localTip: TopicAnchorTip, + remoteTip: TopicAnchorTip, + report: BASMPeerSyncReport + ): Promise { + const tipsMatch = localTip.tac === remoteTip.tac && localTip.blockHeight === remoteTip.blockHeight + if (localTip.blockHeight >= 0 && tipsMatch) { + const localAnchor = await this.storage.findTopicBlockAnchor?.(topic, localTip.blockHeight) + await this.requireCanonicalBASMAnchor( + requireBASMDefined(localAnchor, localAnchor?.tac === localTip.tac, 'Local BASM tip lacks its anchor') + ) + } + report.status = tipsMatch ? 'matched' : 'diverged' + report.message = tipsMatch + ? 'Topic anchor tips match' + : 'Remote tip is not ahead; historical divergence needs manual or binary-search reconciliation' + return report + } + + private requireBASMRangePrefix( + localTip: TopicAnchorTip, + remoteTip: TopicAnchorTip, + fromHeight: number, + toHeight: number, + anchors: TopicBlockAnchor[] + ): void { + requireBASM(anchors.length > 0 && anchors.at(-1)?.blockHeight === toHeight, 'BASM range omits its requested target') + requireBASM(localTip.blockHeight < 0 || anchors[0].blockHeight === fromHeight, 'BASM range omits its next height') + let previousTac = localTip.tac + for (const anchor of anchors) { + requireBASM(anchor.tac === computeTac(previousTac, anchor.blockHash, anchor.basmRoot), 'BASM range TAC is inconsistent with its prefix') + previousTac = anchor.tac + } + if (toHeight === remoteTip.blockHeight) requireBASM(previousTac === remoteTip.tac, 'BASM range differs from its tip') + } + + private async advanceBASMWithRemoteAnchors( + topic: string, + remote: BASMRemote, + localTip: TopicAnchorTip, + remoteTip: TopicAnchorTip, + report: BASMPeerSyncReport + ): Promise { + const fromHeight = localTip.blockHeight < 0 + ? Math.max(remoteTip.blockHeight - DEFAULT_BASM_SYNC_PAGE_SIZE + 1, 0) + : localTip.blockHeight + 1 + const toHeight = Math.min(fromHeight + DEFAULT_BASM_SYNC_PAGE_SIZE - 1, remoteTip.blockHeight) + const range = await remote.requestTopicAnchorRange(fromHeight, toHeight) + this.requireBASMRangePrefix(localTip, remoteTip, fromHeight, toHeight, range.anchors) + for (const remoteAnchor of range.anchors) { + await this.reconcileRemoteAnchor(topic, remote, remoteAnchor, report) + if (report.status === 'diverged') return report + } + + const finalRemoteTip = await remote.requestTopicAnchorTip() + requireBASM(finalRemoteTip.blockHeight === remoteTip.blockHeight && finalRemoteTip.tac === remoteTip.tac, 'BASM peer history changed during reconciliation') + + const refreshedTip = await this.provideTopicAnchorTip(topic) + report.localTip = refreshedTip + report.status = refreshedTip.blockHeight === remoteTip.blockHeight && refreshedTip.tac === remoteTip.tac ? 'matched' : 'advanced' + return report + } + + private markBASMPeerSyncError( + report: BASMPeerSyncReport, + topic: string, + endpoint: string, + error: unknown + ): BASMPeerSyncReport { + report.status = 'error' + if (error instanceof Error && 'code' in error && typeof error.code === 'string') report.errorCode = error.code + report.message = error instanceof Error ? error.message : String(error) + this.logger.error(`[BASM SYNC] Sync failed for topic "${topic}" with peer "${endpoint}"`, error) + return report + } + private async requireCanonicalBASMAnchor(anchor: TopicBlockAnchor, proofRoot?: string): Promise { if (this.chainTracker === 'scripts only' || this.topicAnchorHeaderResolver === undefined) { throw new Error('BASM reconciliation requires a ChainTracker and canonical header resolver') } - const header = await this.topicAnchorHeaderResolver(anchor.blockHeight) - requireBASM(header !== undefined && header.blockHeight === anchor.blockHeight, 'BASM canonical header is unavailable or has the wrong height') + const resolvedHeader = await this.topicAnchorHeaderResolver(anchor.blockHeight) + const header = requireBASMDefined( + resolvedHeader, + resolvedHeader?.blockHeight === anchor.blockHeight, + 'BASM canonical header is unavailable or has the wrong height' + ) requireBASM(basmHash(header.blockHash.toLowerCase(), 'canonical block hash') === anchor.blockHash, 'BASM anchor block hash is not canonical') if (proofRoot !== undefined && header.merkleRoot !== undefined) { requireBASM(header.merkleRoot.toLowerCase() === proofRoot, 'BASM proof root differs from its canonical header') @@ -1839,7 +1895,7 @@ export class Engine { } for (const { txid, blockIndex } of admitted) { const leaf = compoundPath.path[0]?.find(item => item.hash === txid) - requireBASM(leaf !== undefined && leaf.offset === blockIndex, 'BASM proof does not bind the admitted block index') + requireBASM(leaf?.offset === blockIndex, 'BASM proof does not bind the admitted block index') requireBASM(compoundPath.path[0].length !== 1 || compoundPath.path.length !== 1 || blockIndex === 0, 'BASM singleton proof has a nonzero block index') requireBASM(compoundPath.computeRoot(txid) === proofRoot, 'BASM proof root does not match the admitted transaction') } @@ -1868,9 +1924,10 @@ export class Engine { }) const refreshedAnchor = (await remote.requestTopicAnchorRange(anchor.blockHeight, anchor.blockHeight)).anchors[0] requireBASM( - refreshedAnchor !== undefined && refreshedAnchor.blockHash === anchor.blockHash && - refreshedAnchor.basmRoot === anchor.basmRoot && refreshedAnchor.admittedCount === anchor.admittedCount && - refreshedAnchor.tac === anchor.tac, + refreshedAnchor?.blockHash === anchor.blockHash && + refreshedAnchor?.basmRoot === anchor.basmRoot && + refreshedAnchor?.admittedCount === anchor.admittedCount && + refreshedAnchor?.tac === anchor.tac, 'BASM peer anchor changed before admission' ) const commitHeader = await this.requireCanonicalBASMAnchor(anchor, proofRoot) From 37f5927fe3204ff1bb5fd29f6212190c4b508596 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:12:36 -0500 Subject: [PATCH 019/168] fix(sonar): overlay lookup discovery complexity and style Extract query/session and host-tracking helpers so cognitive complexity stays within Sonar's limit, and clear the remaining overlay-tools findings without changing lookup behavior. --- .../sdk/src/overlay-tools/LookupDiscovery.ts | 89 +- .../sdk/src/overlay-tools/LookupHostQueue.ts | 10 +- .../sdk/src/overlay-tools/LookupResolver.ts | 946 ++++++++++++------ .../src/overlay-tools/LookupResponseReader.ts | 141 +-- 4 files changed, 799 insertions(+), 387 deletions(-) diff --git a/packages/sdk/src/overlay-tools/LookupDiscovery.ts b/packages/sdk/src/overlay-tools/LookupDiscovery.ts index 12dbe6641..8a97eb2ac 100644 --- a/packages/sdk/src/overlay-tools/LookupDiscovery.ts +++ b/packages/sdk/src/overlay-tools/LookupDiscovery.ts @@ -18,6 +18,15 @@ export class LookupDiscovery { private readonly listeners = new Set<(state: LookupDiscoveryUpdate) => void>() private started = false private abandoned = false + private readonly consume = (bytes: number): void => { + if (this.abandoned) throw new LookupResourceLimitError('abandoned') + if (bytes > this.limits.maxTotalBytes - this.state.receivedBytes) { + this.state.limitsHit.add('maxTotalBytes') + throw new LookupResourceLimitError('maxTotalBytes') + } + this.state.receivedBytes += bytes + this.emit() + } constructor( private readonly trackers: string[], @@ -50,49 +59,59 @@ export class LookupDiscovery { for (const listener of this.listeners) listener(this.state) } - private consume = (bytes: number): void => { - if (this.abandoned) throw new LookupResourceLimitError('abandoned') - if (bytes > this.limits.maxTotalBytes - this.state.receivedBytes) { - this.state.limitsHit.add('maxTotalBytes') - throw new LookupResourceLimitError('maxTotalBytes') + private collectTrackerHosts(candidates: string[], share: number): string[] { + const hosts = new Set() + for (const candidate of candidates) { + const host = normalizeLookupHost(candidate) + if (host === null) { + this.state.skippedHosts++ + continue + } + if (hosts.has(host)) continue + if (hosts.size >= share) { + this.state.skippedHosts++ + this.state.limitsHit.add('maxHostsPerTracker') + } else { + hosts.add(host) + } + } + return Array.from(hosts) + } + + private recordTrackerFailure(error: unknown): void { + if (error instanceof LookupResourceLimitError) this.state.limitsHit.add(error.limit) + else if (!this.controller.signal.aborted) this.state.trackersFailed++ + } + + private async processTracker(tracker: string, share: number): Promise { + try { + const candidates = await this.lookup(tracker, this.controller.signal, this.consume) + if (this.abandoned) return + this.state.sources.set(tracker, this.collectTrackerHosts(candidates, share)) + } catch (error) { + this.recordTrackerFailure(error) + } finally { + this.state.trackersCompleted++ + this.emit() + } + } + + private async drainTrackers(share: number, cursor: { value: number }): Promise { + while (!this.controller.signal.aborted && cursor.value < this.trackers.length) { + await this.processTracker(this.trackers[cursor.value++], share) } - this.state.receivedBytes += bytes - this.emit() } private async run(): Promise { - let cursor = 0 + const cursor = { value: 0 } // Each tracker keeps a reserved share, so an early advertisement flood // cannot consume the complete candidate budget before later sources reply. const share = Math.min(this.limits.maxHostsPerTracker, Math.max(1, Math.floor(this.limits.maxHosts / Math.max(1, this.trackers.length)))) - const worker = async (): Promise => { - while (!this.controller.signal.aborted && cursor < this.trackers.length) { - const tracker = this.trackers[cursor++] - try { - const candidates = await this.lookup(tracker, this.controller.signal, this.consume) - if (this.abandoned) return - const hosts = new Set() - for (const candidate of candidates) { - const host = normalizeLookupHost(candidate) - if (host === null) { this.state.skippedHosts++; continue } - if (hosts.has(host)) continue - if (hosts.size >= share) { - this.state.skippedHosts++ - this.state.limitsHit.add('maxHostsPerTracker') - } else hosts.add(host) - } - this.state.sources.set(tracker, Array.from(hosts)) - } catch (error) { - if (error instanceof LookupResourceLimitError) this.state.limitsHit.add(error.limit) - else if (!this.controller.signal.aborted) this.state.trackersFailed++ - } finally { - this.state.trackersCompleted++ - this.emit() - } - } - } - await Promise.all(Array.from({ length: Math.min(this.limits.trackerConcurrency, this.trackers.length) }, worker)) + await Promise.all(Array.from( + { length: Math.min(this.limits.trackerConcurrency, this.trackers.length) }, + () => this.drainTrackers(share, cursor) + )) this.state.done = true this.finish(this.state, this.abandoned) this.emit() diff --git a/packages/sdk/src/overlay-tools/LookupHostQueue.ts b/packages/sdk/src/overlay-tools/LookupHostQueue.ts index 6e5d86cf4..33d0f0d43 100644 --- a/packages/sdk/src/overlay-tools/LookupHostQueue.ts +++ b/packages/sdk/src/overlay-tools/LookupHostQueue.ts @@ -46,11 +46,15 @@ export class LookupHostQueue { private next(): string | undefined { const sources = Array.from(this.queues.values()) - for (let i = 0; i < sources.length; i++) { - const index = this.cursor++ % sources.length + for (const [offset] of sources.entries()) { + const index = (this.cursor + offset) % sources.length const host = sources[index].shift() - if (host !== undefined) return host + if (host !== undefined) { + this.cursor = index + 1 + return host + } } + this.cursor += sources.length return undefined } diff --git a/packages/sdk/src/overlay-tools/LookupResolver.ts b/packages/sdk/src/overlay-tools/LookupResolver.ts index 2d1ccb985..047372ba1 100644 --- a/packages/sdk/src/overlay-tools/LookupResolver.ts +++ b/packages/sdk/src/overlay-tools/LookupResolver.ts @@ -291,6 +291,20 @@ function isFreeformAnswer(value: unknown): value is LookupFreeformAnswer { return answer.type === 'freeform' && Object.hasOwn(answer, 'result') } +function lookupAnswerRetainedBytes(answer: LookupAnswer): number { + let retained = 0 + for (const output of answer.outputs) retained += output.beef.length + (output.context?.length ?? 0) + return retained +} + +function copyLookupOutput(output: LookupAnswer['outputs'][number]): LookupAnswer['outputs'][number] { + return { + ...output, + beef: output.beef.slice(), + ...(output.context === undefined ? {} : { context: output.context.slice() }) + } +} + /** A wall-clock deadline that rejects after `timeoutMs`, optionally aborting a controller. */ interface Deadline { /** Rejects with `Error('Request timed out')` once the timer fires. */ @@ -643,7 +657,7 @@ interface LookupQuerySessionOptions { waitForAllHosts: boolean correlationId?: string limits: LookupLimits - onEvidence?: LookupQueryOptions['onEvidence'] + onEvidence: LookupQueryOptions['onEvidence'] resolveTxId: (output: LookupAnswer['outputs'][number], now: number) => string | null } @@ -824,6 +838,38 @@ interface LookupHostFailureContext { notificationCooldownMs: number } +/** Mutable orchestration state for one raw lookup query. */ +interface LookupQueryRun { + question: LookupQuestion + timeout: number | undefined + options: LookupQueryOptions | undefined + limits: LookupLimits + session: LookupQuerySession + controller: AbortController + queue: LookupHostQueue + seen: Set + sourceQuota: number + discoveryBytes: number + discoverySkipped: number + processedSources: Set + releaseDiscovery: (() => void) | undefined + discoveryFinished: boolean + noHostsError: Error | undefined + cleaned: boolean + timer: ReturnType + abort: () => void + iteratorSignal: AbortSignal +} + +interface LookupDiscoveryPlan { + key: string + cached: LookupHostsCacheEntry | undefined + configuredAdditional: string[] + cacheHasAvailableHost: boolean + refresh: boolean + initialQuota: number +} + /** * Represents a Lookup Resolver. */ @@ -1117,277 +1163,530 @@ export default class LookupResolver { } } - private async *queryProgress(inputQuestion: LookupQuestion, timeout: number | undefined, - options: LookupQueryOptions | undefined, iteratorSignal: AbortSignal): AsyncIterable { - // Capture JSON wire values once, before any discovery or queued host can - // observe a caller's later mutation. Custom non-JSON questions retain their - // historical facilitator-defined semantics when they cannot be cloned. - let question: LookupQuestion - try { question = structuredClone(inputQuestion) } - catch { - if (this.facilitator instanceof HTTPSOverlayLookupFacilitator) question = JSON.parse(stringifyBRC100(inputQuestion)) as LookupQuestion - else question = { ...inputQuestion } + private cloneLookupQuestion(inputQuestion: LookupQuestion): LookupQuestion { + try { + return structuredClone(inputQuestion) + } catch { + if (this.facilitator instanceof HTTPSOverlayLookupFacilitator) { + return JSON.parse(stringifyBRC100(inputQuestion)) as LookupQuestion + } + return { ...inputQuestion } } - const limits = lookupLimits(this.limits, options?.limits, { + } + + private lookupQueryLimits(options: LookupQueryOptions | undefined): LookupLimits { + return lookupLimits(this.limits, options?.limits, { ...(options?.evidenceLimits?.maxOutputs === undefined ? {} : { maxEvidenceOutputs: options.evidenceLimits.maxOutputs }), ...(options?.evidenceLimits?.maxBytes === undefined ? {} : { maxEvidenceBytes: options.evidenceLimits.maxBytes }) }) - const deadlineMs = options?.deadlineMs ?? 10_000 + } + + private assertLookupDeadline(deadlineMs: number): void { if (!Number.isFinite(deadlineMs) || deadlineMs < 0 || deadlineMs > 2_147_483_647) { throw new RangeError('Lookup deadlineMs must be between 0 and 2147483647') } - if (this.activeQueries >= 128) throw new LookupResourceLimitError('activeQueries') - this.activeQueries++ + } + + private lookupQueryStopped(run: LookupQueryRun): boolean { + return run.controller.signal.aborted || run.session.closed + } + + private consumeLookupQueryBytes(run: LookupQueryRun, bytes: number): void { + if (run.controller.signal.aborted) throw lookupAbortError() + if (bytes > run.limits.maxTotalBytes - run.session.receivedBytes) { + run.session.limit('maxTotalBytes') + throw new LookupResourceLimitError('maxTotalBytes') + } + run.session.receivedBytes += bytes + } + + private skipQueuedLookupHosts(run: LookupQueryRun, count: number, limited: boolean): void { + if (count <= 0) return + run.session.skippedHosts += count + if (limited) run.session.limit('maxHosts') + } + + private stopLookupQueryRun(run: LookupQueryRun, reason: 'deadline' | 'cancelled'): void { + if (run.controller.signal.aborted) return + run.session.limit(reason) + run.session.terminalReason = reason + run.session.accepting = false + run.session.discoveryComplete = false + run.controller.abort() + run.releaseDiscovery?.() + run.discoveryFinished = true + run.queue.cancel() + } + + private cleanupLookupQueryRun(run: LookupQueryRun): void { + if (run.cleaned) return + run.cleaned = true + run.session.close() + clearTimeout(run.timer) + run.options?.signal?.removeEventListener('abort', run.abort) + run.iteratorSignal.removeEventListener('abort', run.abort) + run.controller.abort() + run.releaseDiscovery?.() + run.queue.cancel() + this.activeQueries-- + } + + private finishLookupSources(run: LookupQueryRun): void { + run.discoveryFinished = true + run.queue.finishSources() + } + + private lookupQueryNoHostsError(run: LookupQueryRun): Error | undefined { + if (run.session.hostCount !== 0 || run.session.terminalReason !== 'settled') return undefined + if (run.noHostsError !== undefined) return run.noHostsError + return new Error( + `No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${run.question.service}` + ) + } + + private finishLookupQueryRun(run: LookupQueryRun): void { + if (!run.discoveryFinished) return + run.session.finish(this.lookupQueryNoHostsError(run)) + } + + private createLookupQueryRun( + question: LookupQuestion, + timeout: number | undefined, + options: LookupQueryOptions | undefined, + limits: LookupLimits, + iteratorSignal: AbortSignal, + deadlineMs: number + ): LookupQueryRun { const controller = new AbortController() const session = new LookupQuerySession({ - graceMs: options?.graceMs ?? 80, softTimeoutMs: options?.softTimeoutMs, + graceMs: options?.graceMs ?? 80, + softTimeoutMs: options?.softTimeoutMs, waitForAllHosts: options?.waitForAllHosts ?? options?.holdForUnknownHosts ?? false, correlationId: options?.correlationId ?? (this.telemetry.enabled ? this.telemetry.createCorrelationId() : undefined), - limits, onEvidence: options?.onEvidence, resolveTxId: (output, now) => this.resolveTxIdForOutput(output, now) + limits, + onEvidence: options?.onEvidence, + resolveTxId: (output, now) => this.resolveTxIdForOutput(output, now) }) - let releaseDiscovery: (() => void) | undefined - let discoveryFinished = false - let noHostsError: Error | undefined - const seen = new Set() - let sourceQuota = limits.maxHosts - let discoveryBytes = 0 - let discoverySkipped = 0 - const processedSources = new Set() - const consume = (bytes: number): void => { - if (controller.signal.aborted) throw lookupAbortError() - if (bytes > limits.maxTotalBytes - session.receivedBytes) { - session.limit('maxTotalBytes') - throw new LookupResourceLimitError('maxTotalBytes') - } - session.receivedBytes += bytes + let run: LookupQueryRun + run = { + question, + timeout, + options, + limits, + session, + controller, + seen: new Set(), + sourceQuota: limits.maxHosts, + discoveryBytes: 0, + discoverySkipped: 0, + processedSources: new Set(), + releaseDiscovery: undefined, + discoveryFinished: false, + noHostsError: undefined, + cleaned: false, + iteratorSignal, + abort: () => this.stopLookupQueryRun(run, 'cancelled'), + queue: new LookupHostQueue( + limits.maxHosts, + limits.hostConcurrency, + async host => await this.runQueuedLookupHost(run, host), + (count, limited) => this.skipQueuedLookupHosts(run, count, limited) + ), + timer: setTimeout(() => this.stopLookupQueryRun(run, 'deadline'), deadlineMs) } - const queue = new LookupHostQueue(limits.maxHosts, limits.hostConcurrency, async host => { - if (controller.signal.aborted) return - session.hostCount++ - const startedAt = Date.now() - try { - const answer = await this.lookupHostWithTracking(host, question, timeout, controller.signal, { - maxResponseBytes: limits.maxResponseBytes, maxOutputs: limits.maxOutputs, consumeBytes: consume - }) - if (controller.signal.aborted || session.closed) return - let ownedAnswer = answer - if (isOutputListAnswer(answer)) { - let retained = 0 - for (const output of answer.outputs) retained += output.beef.length + (output.context?.length ?? 0) - if (retained > limits.maxTotalBytes - session.retainedBytes) throw new LookupResourceLimitError('maxTotalBytes') - session.retainedBytes += retained - ownedAnswer = { type: 'output-list', outputs: answer.outputs.map(output => ({ - ...output, beef: output.beef.slice(), ...(output.context === undefined ? {} : { context: output.context.slice() }) - })) } - session.receiveEvidence(host, ownedAnswer, options?.onEvidence) - } - if (controller.signal.aborted || session.closed) return - this.recordLookupHostAnswer(session, question.service, host, ownedAnswer, startedAt, session.correlationId) - } catch (error) { - if (controller.signal.aborted || session.closed) return - if (error instanceof LookupResourceLimitError) session.limit(error.limit) - else this.recordLookupHostFailure({ - session, service: question.service, host, hostStartedAt: startedAt, - correlationId: session.correlationId, onUnreachableHost: options?.onUnreachableHost, - notificationCooldownMs: this.unreachableNotificationCooldown(options) - }, error) - } finally { session.recordDone() } - }, (count, limited) => { if (count > 0) { session.skippedHosts += count; if (limited) session.limit('maxHosts') } }) - - const stop = (reason: 'deadline' | 'cancelled'): void => { - if (controller.signal.aborted) return - session.limit(reason) - session.terminalReason = reason - session.accepting = false - session.discoveryComplete = false - controller.abort() - releaseDiscovery?.() - discoveryFinished = true - queue.cancel() - } - const abort = (): void => stop('cancelled') - options?.signal?.addEventListener('abort', abort, { once: true }) - iteratorSignal.addEventListener('abort', abort, { once: true }) - const timer = setTimeout(() => stop('deadline'), deadlineMs) - - let cleaned = false - const cleanup = (): void => { - if (cleaned) return - cleaned = true - session.close() - clearTimeout(timer) - options?.signal?.removeEventListener('abort', abort) - iteratorSignal.removeEventListener('abort', abort) - controller.abort() - releaseDiscovery?.() - queue.cancel() - this.activeQueries-- - } - - const admit = (source: string, candidates: string[]): void => { - if (controller.signal.aborted) return - const hosts: string[] = [] - const scanLimit = Math.min(candidates.length, limits.maxHosts * 4) - if (candidates.length > scanLimit) { session.skippedHosts += candidates.length - scanLimit; session.limit('maxHosts') } - for (let candidateIndex = 0; candidateIndex < scanLimit; candidateIndex++) { - const candidate = candidates[candidateIndex] - const host = normalizeLookupHost(candidate, source === 'configured' || source === 'additional') - if (host === null) { session.skippedHosts++; continue } - if (seen.has(host)) continue - // Keep this dedup set finite, too. Source-level discovery reservations - // protect later trackers before candidates reach this queue. - if (seen.size >= limits.maxHosts) { session.skippedHosts++; session.limit('maxHosts'); continue } - seen.add(host) - session.discoveredHosts++ - hosts.push(host) + options?.signal?.addEventListener('abort', run.abort, { once: true }) + iteratorSignal.addEventListener('abort', run.abort, { once: true }) + return run + } + + private retainLookupHostAnswer( + run: LookupQueryRun, + host: string, + answer: LookupFacilitatorAnswer + ): LookupFacilitatorAnswer { + if (!isOutputListAnswer(answer)) return answer + const retained = lookupAnswerRetainedBytes(answer) + if (retained > run.limits.maxTotalBytes - run.session.retainedBytes) { + throw new LookupResourceLimitError('maxTotalBytes') + } + run.session.retainedBytes += retained + const ownedAnswer: LookupAnswer = { + type: 'output-list', + outputs: answer.outputs.map(copyLookupOutput) + } + run.session.receiveEvidence(host, ownedAnswer, run.options?.onEvidence) + return ownedAnswer + } + + private recordQueuedLookupHostFailure( + run: LookupQueryRun, + host: string, + startedAt: number, + error: unknown + ): void { + if (this.lookupQueryStopped(run)) return + if (error instanceof LookupResourceLimitError) { + run.session.limit(error.limit) + return + } + this.recordLookupHostFailure({ + session: run.session, + service: run.question.service, + host, + hostStartedAt: startedAt, + correlationId: run.session.correlationId, + onUnreachableHost: run.options?.onUnreachableHost, + notificationCooldownMs: this.unreachableNotificationCooldown(run.options) + }, error) + } + + private async settleQueuedLookupHost(run: LookupQueryRun, host: string, startedAt: number): Promise { + const answer = await this.lookupHostWithTracking(host, run.question, run.timeout, run.controller.signal, { + maxResponseBytes: run.limits.maxResponseBytes, + maxOutputs: run.limits.maxOutputs, + consumeBytes: bytes => this.consumeLookupQueryBytes(run, bytes) + }) + if (this.lookupQueryStopped(run)) return + const ownedAnswer = this.retainLookupHostAnswer(run, host, answer) + if (this.lookupQueryStopped(run)) return + this.recordLookupHostAnswer( + run.session, + run.question.service, + host, + ownedAnswer, + startedAt, + run.session.correlationId + ) + } + + private async runQueuedLookupHost(run: LookupQueryRun, host: string): Promise { + if (run.controller.signal.aborted) return + run.session.hostCount++ + const startedAt = Date.now() + try { + await this.settleQueuedLookupHost(run, host, startedAt) + } catch (error) { + this.recordQueuedLookupHostFailure(run, host, startedAt, error) + } finally { + run.session.recordDone() + } + } + + private collectAdmittedLookupHosts(run: LookupQueryRun, source: string, candidates: string[]): string[] { + const hosts: string[] = [] + const scanLimit = Math.min(candidates.length, run.limits.maxHosts * 4) + if (candidates.length > scanLimit) { + run.session.skippedHosts += candidates.length - scanLimit + run.session.limit('maxHosts') + } + const allowParameters = source === 'configured' || source === 'additional' + for (const candidate of candidates.slice(0, scanLimit)) { + const host = normalizeLookupHost(candidate, allowParameters) + if (host === null) { + run.session.skippedHosts++ + continue } - if (hosts.length === 0) return - try { - const available = this.prepareHostsForQuery(hosts, `lookup service ${question.service}`) - session.skippedHosts += hosts.length - available.length - queue.add(source, available) - } catch (error) { - session.skippedHosts += hosts.length - noHostsError = error instanceof Error ? error : new Error(lookupErrorMessage(error)) + if (run.seen.has(host)) continue + if (run.seen.size >= run.limits.maxHosts) { + run.session.skippedHosts++ + run.session.limit('maxHosts') + continue } + run.seen.add(host) + run.session.discoveredHosts++ + hosts.push(host) } - const sourcesDone = (): void => { - discoveryFinished = true - queue.finishSources() - } - const acceptDiscovery = (state: LookupDiscoveryUpdate): void => { - if (controller.signal.aborted) return - session.trackersTotal = state.trackersTotal - session.trackersCompleted = state.trackersCompleted - session.trackersFailed = state.trackersFailed - session.skippedHosts += state.skippedHosts - discoverySkipped - discoverySkipped = state.skippedHosts - for (const name of state.limitsHit) session.limit(name) - try { consume(state.receivedBytes - discoveryBytes) } catch (error) { - if (error instanceof LookupResourceLimitError) session.limit(error.limit) - controller.abort() - releaseDiscovery?.() - queue.cancel() - discoveryFinished = true - return + return hosts + } + + private admitLookupHosts(run: LookupQueryRun, source: string, candidates: string[]): void { + if (run.controller.signal.aborted) return + const hosts = this.collectAdmittedLookupHosts(run, source, candidates) + if (hosts.length === 0) return + try { + const available = this.prepareHostsForQuery(hosts, `lookup service ${run.question.service}`) + run.session.skippedHosts += hosts.length - available.length + run.queue.add(source, available) + } catch (error) { + run.session.skippedHosts += hosts.length + run.noHostsError = error instanceof Error ? error : new Error(lookupErrorMessage(error)) + } + } + + private admitQuotaLimitedLookupHosts( + run: LookupQueryRun, + source: string, + hosts: string[], + quota: number, + limitName: string + ): void { + this.admitLookupHosts(run, source, hosts.slice(0, quota)) + if (hosts.length > quota) { + run.session.skippedHosts += hosts.length - quota + run.session.limit(limitName) + } + } + + private configuredLookupHosts(question: LookupQuestion): string[] { + if (question.service === 'ls_slap') { + if (this.networkPreset === 'local') return ['http://localhost:8080'] + return this.slapTrackers + } + return this.hostOverrides[question.service] ?? ['http://localhost:8080'] + } + + private admitConfiguredLookupSources(run: LookupQueryRun): void { + this.admitLookupHosts(run, 'configured', this.configuredLookupHosts(run.question)) + this.admitLookupHosts(run, 'additional', this.additionalHosts[run.question.service] ?? []) + run.session.discoveryComplete = true + this.finishLookupSources(run) + } + + private lookupCacheHasAvailableHost(cached: LookupHostsCacheEntry | undefined): boolean { + if (cached === undefined) return false + return cached.hosts.some(host => (this.hostReputation.snapshot(host)?.backoffUntil ?? 0) <= Date.now()) + } + + private planLookupDiscovery(run: LookupQueryRun): LookupDiscoveryPlan { + const cached = this.hostsCache.get(run.question.service) + const configuredAdditional = this.additionalHosts[run.question.service] ?? [] + const cacheHasAvailableHost = this.lookupCacheHasAvailableHost(cached) + const cacheCoversCaller = lookupDiscoveryCovers(cached, run.limits) + const cacheFresh = cached !== undefined && cached.expiresAt > Date.now() + const key = lookupDiscoveryCacheKey(run.question.service, run.limits) + const discovery = this.hostsInFlight.get(key) + const refresh = + discovery !== undefined || + cached === undefined || + !cacheCoversCaller || + !cacheFresh || + !cacheHasAvailableHost + const initialSources = + Number(cached !== undefined && cacheHasAvailableHost) + Number(configuredAdditional.length > 0) + const trackerShare = Math.max(1, Math.min(this.slapTrackers.length, run.limits.maxTrackers)) + const initialQuota = refresh + ? Math.max(1, Math.floor(run.limits.maxHosts / (initialSources + trackerShare))) + : run.limits.maxHosts + return { key, cached, configuredAdditional, cacheHasAvailableHost, refresh, initialQuota } + } + + private reuseCachedLookupDiscovery(run: LookupQueryRun, cached: LookupHostsCacheEntry | undefined): void { + run.session.discoveryComplete = cached?.discoveryComplete ?? true + run.session.trackersFailed = cached?.trackersFailed ?? 0 + for (const name of cached?.limitsHit ?? []) run.session.limit(name) + this.finishLookupSources(run) + } + + private selectSlapTrackers(run: LookupQueryRun): { trackers: string[], normalized: string[] } { + const scan = Math.min(this.slapTrackers.length, run.limits.maxTrackers) + const selected = Array.from( + { length: scan }, + (_unused, offset) => this.slapTrackers[(this.trackerCursor + offset) % this.slapTrackers.length] + ) + this.trackerCursor = (this.trackerCursor + scan) % Math.max(1, this.slapTrackers.length) + const normalized = Array.from(new Set(selected.map(host => normalizeLookupHost(host)).filter((host): host is string => host !== null))) + try { + return { + trackers: this.prepareHostsForQuery(normalized.slice(0, run.limits.maxTrackers), 'SLAP trackers'), + normalized } - discoveryBytes = state.receivedBytes - for (const [source, hosts] of state.sources) { - if (processedSources.has(source)) continue - processedSources.add(source) - admit(source, hosts.slice(0, sourceQuota)) - if (hosts.length > sourceQuota) { - session.skippedHosts += hosts.length - sourceQuota - session.limit('maxHostsPerTracker') - } + } catch (error) { + run.noHostsError = error instanceof Error ? error : new Error(lookupErrorMessage(error)) + return { trackers: [], normalized } + } + } + + private async lookupSlapTrackerHosts( + run: LookupQueryRun, + tracker: string, + signal: AbortSignal, + charge: (bytes: number) => void + ): Promise { + const answer = await this.lookupHostWithTracking( + tracker, + { service: 'ls_slap', query: { service: run.question.service } }, + MAX_TRACKER_WAIT_TIME, + signal, + { + maxResponseBytes: run.limits.maxResponseBytes, + maxOutputs: run.limits.maxOutputs, + consumeBytes: charge } - session.discoveryComplete = state.done && state.trackersFailed === 0 && state.limitsHit.size === 0 && state.skippedHosts === 0 - session.wake() - if (state.done) sourcesDone() + ) + const hosts = isOutputListAnswer(answer) ? this.extractHostsFromAnswer(answer, run.question.service) : [] + for (const host of hosts) { + if (this.advertisedBy.size >= this.hostsMaxEntries * run.limits.maxHosts) this.evictOldest(this.advertisedBy) + this.advertisedBy.set(host, tracker) } + return hosts + } + + private completeLookupDiscoveryRefresh( + run: LookupQueryRun, + key: string, + discovery: LookupDiscovery, + state: LookupDiscoveryUpdate, + abandoned: boolean + ): void { + if (this.hostsInFlight.get(key) !== discovery) return + this.hostsInFlight.delete(key) + if (abandoned) return + const hosts = Array.from(new Set(Array.from(state.sources.values()).flat())).slice(0, run.limits.maxHosts) + this.rememberDiscoveredHosts(run.question.service, hosts, run.limits, state) + } + + private createLookupDiscovery(run: LookupQueryRun, key: string): LookupDiscovery { + const selected = this.selectSlapTrackers(run) + let discovery: LookupDiscovery + discovery = new LookupDiscovery( + selected.trackers, + run.limits, + async (tracker, signal, charge) => await this.lookupSlapTrackerHosts(run, tracker, signal, charge), + (state, abandoned) => this.completeLookupDiscoveryRefresh(run, key, discovery, state, abandoned) + ) + if (this.slapTrackers.length > run.limits.maxTrackers) discovery.state.limitsHit.add('maxTrackers') + if ( + selected.normalized.length !== this.slapTrackers.length || + selected.trackers.length < Math.min(selected.normalized.length, run.limits.maxTrackers) + ) { + discovery.state.skippedHosts += this.slapTrackers.length - selected.trackers.length + } + return discovery + } + + private refreshLookupDiscovery(run: LookupQueryRun, plan: LookupDiscoveryPlan): void { + run.sourceQuota = Math.max( + 1, + Math.floor( + (run.limits.maxHosts - run.seen.size) / + Math.max(1, Math.min(this.slapTrackers.length, run.limits.maxTrackers)) + ) + ) + let discovery = this.hostsInFlight.get(plan.key) + if (discovery === undefined) { + discovery = this.createLookupDiscovery(run, plan.key) + this.hostsInFlight.set(plan.key, discovery) + } + run.releaseDiscovery = discovery.subscribe(state => this.acceptLookupDiscovery(run, state)) + if (run.controller.signal.aborted) run.releaseDiscovery() + } + + private admitDiscoveredLookupSources(run: LookupQueryRun): void { + const plan = this.planLookupDiscovery(run) + if (plan.cached !== undefined && plan.cacheHasAvailableHost) { + this.admitQuotaLimitedLookupHosts(run, 'cache', plan.cached.hosts, plan.initialQuota, 'maxHosts') + } + if (plan.configuredAdditional.length > 0) { + this.admitQuotaLimitedLookupHosts(run, 'additional', plan.configuredAdditional, plan.initialQuota, 'maxHosts') + } + if (plan.refresh) this.refreshLookupDiscovery(run, plan) + else this.reuseCachedLookupDiscovery(run, plan.cached) + } + + private admitLookupSources(run: LookupQueryRun): void { + if ( + run.question.service === 'ls_slap' || + this.hostOverrides[run.question.service] != null || + this.networkPreset === 'local' + ) { + this.admitConfiguredLookupSources(run) + return + } + this.admitDiscoveredLookupSources(run) + } + + private syncLookupDiscoveryProgress(run: LookupQueryRun, state: LookupDiscoveryUpdate): void { + run.session.trackersTotal = state.trackersTotal + run.session.trackersCompleted = state.trackersCompleted + run.session.trackersFailed = state.trackersFailed + run.session.skippedHosts += state.skippedHosts - run.discoverySkipped + run.discoverySkipped = state.skippedHosts + for (const name of state.limitsHit) run.session.limit(name) + } + + private chargeLookupDiscoveryBytes(run: LookupQueryRun, state: LookupDiscoveryUpdate): boolean { try { - this.telemetry.capture({ name: 'sdk.overlay.lookup.started', component: 'sdk.lookup-resolver', - severity: 'debug', correlationId: session.correlationId, - attributes: { service: question.service, network: this.networkPreset, hostCount: 0 } }) - if (options?.signal?.aborted === true || iteratorSignal.aborted) abort() - if (!controller.signal.aborted) { - if (question.service === 'ls_slap' || this.hostOverrides[question.service] != null || this.networkPreset === 'local') { - const direct = question.service === 'ls_slap' - ? (this.networkPreset === 'local' ? ['http://localhost:8080'] : this.slapTrackers) - : this.hostOverrides[question.service] ?? ['http://localhost:8080'] - admit('configured', direct) - admit('additional', this.additionalHosts[question.service] ?? []) - session.discoveryComplete = true - sourcesDone() - } else { - const cached = this.hostsCache.get(question.service) - const configuredAdditional = this.additionalHosts[question.service] ?? [] - const cacheHasAvailableHost = - cached?.hosts.some(host => (this.hostReputation.snapshot(host)?.backoffUntil ?? 0) <= Date.now()) ?? - false - const cacheCoversCaller = lookupDiscoveryCovers(cached, limits) - const cacheFresh = cached !== undefined && cached.expiresAt > Date.now() - const key = lookupDiscoveryCacheKey(question.service, limits) - let discovery = this.hostsInFlight.get(key) - const refresh = - discovery !== undefined || - cached === undefined || - !cacheCoversCaller || - !cacheFresh || - !cacheHasAvailableHost - const initialSources = - Number(cached !== undefined && cacheHasAvailableHost) + Number(configuredAdditional.length > 0) - const initialQuota = refresh ? Math.max(1, Math.floor(limits.maxHosts / (initialSources + Math.max(1, Math.min(this.slapTrackers.length, limits.maxTrackers))))) : limits.maxHosts - if (cached !== undefined && cacheHasAvailableHost) { - // Reserve a source share for cached membership and each late tracker. - const cachedLimit = initialQuota - admit('cache', cached.hosts.slice(0, cachedLimit)) - if (cached.hosts.length > cachedLimit) { session.skippedHosts += cached.hosts.length - cachedLimit; session.limit('maxHosts') } - } - if (configuredAdditional.length > 0) { - admit('additional', configuredAdditional.slice(0, initialQuota)) - if (configuredAdditional.length > initialQuota) { session.skippedHosts += configuredAdditional.length - initialQuota; session.limit('maxHosts') } - } - if (refresh) { - sourceQuota = Math.max(1, Math.floor((limits.maxHosts - seen.size) / Math.max(1, Math.min(this.slapTrackers.length, limits.maxTrackers)))) - if (discovery === undefined) { - let trackers: string[] = [] - const selected: string[] = [] - const scan = Math.min(this.slapTrackers.length, limits.maxTrackers) - for (let i = 0; i < scan; i++) selected.push(this.slapTrackers[(this.trackerCursor + i) % this.slapTrackers.length]) - this.trackerCursor = (this.trackerCursor + scan) % Math.max(1, this.slapTrackers.length) - const normalized = Array.from(new Set(selected.map(host => normalizeLookupHost(host)).filter((host): host is string => host !== null))) - try { trackers = this.prepareHostsForQuery(normalized.slice(0, limits.maxTrackers), 'SLAP trackers') } - catch (error) { noHostsError = error instanceof Error ? error : new Error(lookupErrorMessage(error)) } - discovery = new LookupDiscovery(trackers, limits, async (tracker, signal, charge) => { - const answer = await this.lookupHostWithTracking(tracker, { service: 'ls_slap', query: { service: question.service } }, MAX_TRACKER_WAIT_TIME, signal, - { maxResponseBytes: limits.maxResponseBytes, maxOutputs: limits.maxOutputs, consumeBytes: charge }) - const hosts = isOutputListAnswer(answer) ? this.extractHostsFromAnswer(answer, question.service) : [] - for (const host of hosts) { - if (this.advertisedBy.size >= this.hostsMaxEntries * limits.maxHosts) this.evictOldest(this.advertisedBy) - this.advertisedBy.set(host, tracker) - } - return hosts - }, (state, abandoned) => { - if (this.hostsInFlight.get(key) !== discovery) return - this.hostsInFlight.delete(key) - if (abandoned) return - const hosts = Array.from(new Set(Array.from(state.sources.values()).flat())).slice(0, limits.maxHosts) - this.rememberDiscoveredHosts(question.service, hosts, limits, state) - }) - if (this.slapTrackers.length > limits.maxTrackers) discovery.state.limitsHit.add('maxTrackers') - if (normalized.length !== this.slapTrackers.length || trackers.length < Math.min(normalized.length, limits.maxTrackers)) { - discovery.state.skippedHosts += this.slapTrackers.length - trackers.length - } - this.hostsInFlight.set(key, discovery) - } - releaseDiscovery = discovery.subscribe(acceptDiscovery) - if (controller.signal.aborted) releaseDiscovery() - } else { - session.discoveryComplete = cached?.discoveryComplete ?? true - session.trackersFailed = cached?.trackersFailed ?? 0 - for (const name of cached?.limitsHit ?? []) session.limit(name) - sourcesDone() - } - } - } else queue.cancel() - void queue.done.then(() => { - const error = session.hostCount === 0 && session.terminalReason === 'settled' - ? noHostsError ?? new Error(`No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${question.service}`) - : undefined - if (discoveryFinished) session.finish(error) - }) - for await (const progress of session.progress()) { + this.consumeLookupQueryBytes(run, state.receivedBytes - run.discoveryBytes) + } catch (error) { + if (error instanceof LookupResourceLimitError) run.session.limit(error.limit) + run.controller.abort() + run.releaseDiscovery?.() + run.queue.cancel() + run.discoveryFinished = true + return false + } + run.discoveryBytes = state.receivedBytes + return true + } + + private admitLookupDiscoverySources(run: LookupQueryRun, state: LookupDiscoveryUpdate): void { + for (const [source, hosts] of state.sources) { + if (run.processedSources.has(source)) continue + run.processedSources.add(source) + this.admitQuotaLimitedLookupHosts(run, source, hosts, run.sourceQuota, 'maxHostsPerTracker') + } + } + + private acceptLookupDiscovery(run: LookupQueryRun, state: LookupDiscoveryUpdate): void { + if (run.controller.signal.aborted) return + this.syncLookupDiscoveryProgress(run, state) + if (!this.chargeLookupDiscoveryBytes(run, state)) return + this.admitLookupDiscoverySources(run, state) + run.session.discoveryComplete = + state.done && + state.trackersFailed === 0 && + state.limitsHit.size === 0 && + state.skippedHosts === 0 + run.session.wake() + if (state.done) this.finishLookupSources(run) + } + + private beginLookupQueryRun(run: LookupQueryRun): void { + this.telemetry.capture({ + name: 'sdk.overlay.lookup.started', + component: 'sdk.lookup-resolver', + severity: 'debug', + correlationId: run.session.correlationId, + attributes: { service: run.question.service, network: this.networkPreset, hostCount: 0 } + }) + if (run.options?.signal?.aborted === true || run.iteratorSignal.aborted) { + this.stopLookupQueryRun(run, 'cancelled') + } + if (!run.controller.signal.aborted) this.admitLookupSources(run) + else run.queue.cancel() + void run.queue.done.then(() => this.finishLookupQueryRun(run)) + } + + private async *queryProgress( + inputQuestion: LookupQuestion, + timeout: number | undefined, + options: LookupQueryOptions | undefined, + iteratorSignal: AbortSignal + ): AsyncIterable { + // Capture JSON wire values once, before any discovery or queued host can + // observe a caller's later mutation. Custom non-JSON questions retain their + // historical facilitator-defined semantics when they cannot be cloned. + const question = this.cloneLookupQuestion(inputQuestion) + const limits = this.lookupQueryLimits(options) + const deadlineMs = options?.deadlineMs ?? 10_000 + this.assertLookupDeadline(deadlineMs) + if (this.activeQueries >= 128) throw new LookupResourceLimitError('activeQueries') + this.activeQueries++ + const run = this.createLookupQueryRun(question, timeout, options, limits, iteratorSignal, deadlineMs) + try { + this.beginLookupQueryRun(run) + for await (const progress of run.session.progress()) { if (progress.isFinal) { - this.captureLookupCompletedTelemetry(question.service, progress, Date.now() - session.startedAt) - cleanup() + this.captureLookupCompletedTelemetry(question.service, progress, Date.now() - run.session.startedAt) + this.cleanupLookupQueryRun(run) } yield progress } - } finally { cleanup() } + } finally { + this.cleanupLookupQueryRun(run) + } } - /** * Extracts competent host domains from a SLAP tracker response. */ @@ -1501,6 +1800,105 @@ export default class LookupResolver { ) } + private effectiveLookupTimeout(timeout: number | undefined): number { + if (typeof timeout === 'number' && Number.isFinite(timeout) && timeout >= 0) return timeout + return DEFAULT_LOOKUP_TIMEOUT + } + + private startTrackedLookup(args: { + host: string + question: LookupQuestion + timeout: number | undefined + signal: AbortSignal | undefined + controller: AbortController + options: LookupRequestOptions | undefined + reported: { bytes: number } + }): Promise { + const { host, question, timeout, signal, controller, options, reported } = args + const requestOptions = { + ...options, + consumeBytes: (bytes: number): void => { + options?.consumeBytes?.(bytes) + reported.bytes += bytes + } + } + // Start the custom facilitator in a promise chain so synchronous throws + // become rejections governed by the same wall-clock deadline. + const lookupPromise = Promise.resolve().then(() => { + if (signal?.aborted === true) return Promise.reject(lookupAbortError()) + return this.facilitator.lookup(host, question, timeout, controller.signal, requestOptions) + }) + lookupPromise.catch(() => { + /* deadline may win while custom facilitator settles later */ + }) + return lookupPromise + } + + private assertTrackedOutputBudget( + answer: LookupAnswer, + options: LookupRequestOptions | undefined, + reportedBytes: number + ): void { + let bytes = 0 + for (const output of answer.outputs) { + bytes += output.beef.length + (output.context?.length ?? 0) + if (bytes > (options?.maxResponseBytes ?? DEFAULT_LOOKUP_LIMITS.maxResponseBytes)) { + throw new LookupResourceLimitError('maxResponseBytes') + } + } + if (reportedBytes === 0) options?.consumeBytes?.(bytes) + } + + private assertTrackedLookupAnswer( + answer: LookupFacilitatorAnswer, + options: LookupRequestOptions | undefined, + reportedBytes: number + ): void { + if (answer?.type !== 'output-list' || !Array.isArray(answer.outputs)) return + if (answer.outputs.length > (options?.maxOutputs ?? DEFAULT_LOOKUP_LIMITS.maxOutputs)) { + throw new LookupResourceLimitError('maxOutputs') + } + if (!isOutputListAnswer(answer)) return + this.assertTrackedOutputBudget(answer, options, reportedBytes) + } + + private completeTrackedLookup( + host: string, + answer: LookupFacilitatorAnswer, + startedAt: number, + reportedBytes: number, + options: LookupRequestOptions | undefined, + signal: AbortSignal | undefined + ): LookupFacilitatorAnswer { + if (signal?.aborted === true) throw lookupAbortError() + this.assertTrackedLookupAnswer(answer, options, reportedBytes) + if (isOutputListAnswer(answer)) { + this.hostReputation.recordSuccess(host, Date.now() - startedAt) + return answer + } + // A valid freeform response is neutral: it proves this request reached the + // service, but it must not erase an availability backoff established by a + // concurrent failing request and cannot contribute to output aggregation. + if (isFreeformAnswer(answer)) return answer + const malformed = new Error('Malformed lookup response') + this.hostReputation.recordFailure(host, malformed) + throw malformed + } + + private throwTrackedLookupFailure( + host: string, + err: unknown, + signal: AbortSignal | undefined, + deadline: Deadline + ): never { + if (signal?.aborted === true) throw lookupAbortError() + if (err instanceof LookupResourceLimitError) throw err + if (isSemanticLookupRejection(err)) throw err + const normalized = normalizeLookupError(err, deadline.didTimeOut()) + this.hostReputation.recordFailure(host, normalized) + throw normalized + } + private async lookupHostWithTracking( host: string, question: LookupQuestion, @@ -1509,63 +1907,31 @@ export default class LookupResolver { options?: LookupRequestOptions ): Promise { const startedAt = Date.now() - const effectiveTimeout = - typeof timeout === 'number' && Number.isFinite(timeout) && timeout >= 0 - ? timeout - : DEFAULT_LOOKUP_TIMEOUT const controller = new AbortController() const abort = (): void => controller.abort() signal?.addEventListener('abort', abort, { once: true }) - const deadline = createDeadline(effectiveTimeout, controller) - let reportedBytes = 0 - const requestOptions = { ...options, consumeBytes: (bytes: number): void => { - options?.consumeBytes?.(bytes); reportedBytes += bytes - } } - // Start the custom facilitator in a promise chain so synchronous throws - // become rejections governed by the same wall-clock deadline. - const lookupPromise = Promise.resolve().then(() => - signal?.aborted === true ? Promise.reject(lookupAbortError()) : this.facilitator.lookup(host, question, timeout, controller.signal, requestOptions) - ) - lookupPromise.catch(() => { - /* deadline may win while custom facilitator settles later */ + const deadline = createDeadline(this.effectiveLookupTimeout(timeout), controller) + const reported = { bytes: 0 } + const lookupPromise = this.startTrackedLookup({ + host, + question, + timeout, + signal, + controller, + options, + reported }) - let answer: LookupFacilitatorAnswer try { answer = await withLookupAbort(Promise.race([lookupPromise, deadline.promise]), signal) } catch (err) { - if (signal?.aborted === true) throw lookupAbortError() - if (err instanceof LookupResourceLimitError) throw err - const normalized = normalizeLookupError(err, deadline.didTimeOut()) - if (!isSemanticLookupRejection(err)) this.hostReputation.recordFailure(host, normalized) - throw isSemanticLookupRejection(err) ? err : normalized + this.throwTrackedLookupFailure(host, err, signal, deadline) } finally { deadline.cancel() signal?.removeEventListener('abort', abort) controller.abort() } - - if (signal?.aborted === true) throw lookupAbortError() - if (answer != null && answer.type === 'output-list' && Array.isArray(answer.outputs) && answer.outputs.length > (options?.maxOutputs ?? DEFAULT_LOOKUP_LIMITS.maxOutputs)) throw new LookupResourceLimitError('maxOutputs') - if (isOutputListAnswer(answer)) { - let bytes = 0 - for (const output of answer.outputs) { - bytes += output.beef.length + (output.context?.length ?? 0) - if (bytes > (options?.maxResponseBytes ?? DEFAULT_LOOKUP_LIMITS.maxResponseBytes)) throw new LookupResourceLimitError('maxResponseBytes') - } - if (reportedBytes === 0) options?.consumeBytes?.(bytes) - this.hostReputation.recordSuccess(host, Date.now() - startedAt) - return answer - } - - // A valid freeform response is neutral: it proves this request reached the - // service, but it must not erase an availability backoff established by a - // concurrent failing request and cannot contribute to output aggregation. - if (isFreeformAnswer(answer)) return answer - - const malformed = new Error('Malformed lookup response') - this.hostReputation.recordFailure(host, malformed) - throw malformed + return this.completeTrackedLookup(host, answer, startedAt, reported.bytes, options, signal) } private captureHostTelemetry( diff --git a/packages/sdk/src/overlay-tools/LookupResponseReader.ts b/packages/sdk/src/overlay-tools/LookupResponseReader.ts index fe0ed7b4b..7ef119add 100644 --- a/packages/sdk/src/overlay-tools/LookupResponseReader.ts +++ b/packages/sdk/src/overlay-tools/LookupResponseReader.ts @@ -27,7 +27,7 @@ function assertDeclaredLengthIsWithinLimit(response: Response, maxResponseBytes: const normalized = contentLength.trim() // Content-Length is decimal bytes. Treat malformed fields as unknown rather // than accidentally accepting a notation such as "1e6". - if (!/^[0-9]+$/.test(normalized)) return + if (!/^\d+$/.test(normalized)) return const declaredLength = Number(normalized) if (!Number.isSafeInteger(declaredLength) || declaredLength > maxResponseBytes) { @@ -53,14 +53,12 @@ async function readWithAbort( const onAbort = (): void => finish(() => reject(abortReason(signal))) signal.addEventListener('abort', onAbort, { once: true }) - try { - Promise.resolve(reader.read()).then( + Promise.resolve() + .then(() => reader.read()) + .then( result => finish(() => resolve(result)), error => finish(() => reject(error)) ) - } catch (error) { - finish(() => reject(error)) - } // Do not miss an abort that happened while registering the listener. if (signal.aborted) onAbort() @@ -68,11 +66,9 @@ async function readWithAbort( } function cleanUpFailedRead(reader: ReadableStreamDefaultReader, reason: unknown): void { - try { - void Promise.resolve(reader.cancel(reason)).catch(() => undefined) - } catch { - // A broken stream implementation must not replace the response read error. - } + Promise.resolve() + .then(() => reader.cancel(reason)) + .catch(() => undefined) try { reader.releaseLock() @@ -112,31 +108,55 @@ async function yieldAfterReadIfNeeded( if (signal?.aborted) throw abortReason(signal) } -/** - * Reads a lookup response incrementally while enforcing a per-response bound. - * - * This deliberately does not use Response.text(), json(), or arrayBuffer(), - * because those APIs buffer the complete body before a limit can be enforced. - */ -export async function readLookupResponseBytes( - response: Response, +async function accumulateLookupResponseChunk( + bytes: Uint8Array, + totalLength: number, + value: Uint8Array, + readOperations: number, options: LookupResponseReaderOptions -): Promise { +): Promise<{ bytes: Uint8Array, totalLength: number }> { const { signal, maxResponseBytes, consumeBytes } = options - assertValidMaximum(maxResponseBytes) + if (value.byteLength === 0) { + // An eagerly fulfilled read() still schedules only microtasks. Yielding + // periodically lets timers deliver cancellation for endless empty input. + await yieldAfterReadIfNeeded(readOperations, signal) + return { bytes, totalLength } + } - const body = response.body - if (body === null) { - assertDeclaredLengthIsWithinLimit(response, maxResponseBytes) - if (signal?.aborted === true) throw abortReason(signal) - return new Uint8Array(0) + if (value.byteLength > maxResponseBytes - totalLength) { + throw new LookupResourceLimitError('maxResponseBytes') } - const reader = body.getReader() + consumeBytes?.(value.byteLength) + const nextLength = totalLength + value.byteLength + const expanded = expandedBuffer(bytes, nextLength, maxResponseBytes) + // Streams are allowed to reuse a producer-owned Uint8Array. Copy each + // accepted chunk now instead of retaining a mutable producer reference. + expanded.set(value, totalLength) + // Copy before yielding: a producer may reuse or mutate its buffer while + // the task queue runs. + await yieldAfterReadIfNeeded(readOperations, signal) + return { bytes: expanded, totalLength: nextLength } +} + +function releaseLookupResponseReader(reader: ReadableStreamDefaultReader): void { + try { + reader.releaseLock() + } catch { + // A nonstandard stream may have released its lock itself. + } +} + +async function readLookupResponseStream( + reader: ReadableStreamDefaultReader, + response: Response, + options: LookupResponseReaderOptions +): Promise { + const { signal } = options let succeeded = false let failure: unknown try { - assertDeclaredLengthIsWithinLimit(response, maxResponseBytes) + assertDeclaredLengthIsWithinLimit(response, options.maxResponseBytes) if (signal?.aborted === true) throw abortReason(signal) let bytes: Uint8Array = new Uint8Array(0) @@ -146,28 +166,15 @@ export async function readLookupResponseBytes( const { done, value } = await readWithAbort(reader, signal) readOperations++ if (done) break - - if (value.byteLength === 0) { - // An eagerly fulfilled read() still schedules only microtasks. Yielding - // periodically lets timers deliver cancellation for endless empty input. - await yieldAfterReadIfNeeded(readOperations, signal) - continue - } - - if (value.byteLength > maxResponseBytes - totalLength) { - throw new LookupResourceLimitError('maxResponseBytes') - } - - consumeBytes?.(value.byteLength) - const nextLength = totalLength + value.byteLength - bytes = expandedBuffer(bytes, nextLength, maxResponseBytes) - // Streams are allowed to reuse a producer-owned Uint8Array. Copy each - // accepted chunk now instead of retaining a mutable producer reference. - bytes.set(value, totalLength) - totalLength = nextLength - // Copy before yielding: a producer may reuse or mutate its buffer while - // the task queue runs. - await yieldAfterReadIfNeeded(readOperations, signal) + const next = await accumulateLookupResponseChunk( + bytes, + totalLength, + value ?? new Uint8Array(0), + readOperations, + options + ) + bytes = next.bytes + totalLength = next.totalLength } succeeded = true @@ -176,14 +183,30 @@ export async function readLookupResponseBytes( failure = error throw error } finally { - if (succeeded) { - try { - reader.releaseLock() - } catch { - // A nonstandard stream may have released its lock itself. - } - } else { - cleanUpFailedRead(reader, failure) - } + if (succeeded) releaseLookupResponseReader(reader) + else cleanUpFailedRead(reader, failure) } } + +/** + * Reads a lookup response incrementally while enforcing a per-response bound. + * + * This deliberately does not use Response.text(), json(), or arrayBuffer(), + * because those APIs buffer the complete body before a limit can be enforced. + */ +export async function readLookupResponseBytes( + response: Response, + options: LookupResponseReaderOptions +): Promise { + const { signal, maxResponseBytes } = options + assertValidMaximum(maxResponseBytes) + + const body = response.body + if (body === null) { + assertDeclaredLengthIsWithinLimit(response, maxResponseBytes) + if (signal?.aborted === true) throw abortReason(signal) + return new Uint8Array(0) + } + + return await readLookupResponseStream(body.getReader(), response, options) +} From f2c6e48e56ef1d6ee69de2efb1d0373d4617daa4 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:14:06 -0500 Subject: [PATCH 020/168] fix(sonar): overlay mongo and admission findings Brand StorageUint64, extract high-complexity helpers, and apply Sonar-preferred optional chaining, regex, clone, and sort APIs without changing payload identity or uint32 $toLong validators. --- .../src/__tests/AdmissionStorage.test.ts | 8 + .../admission/AdmissionStorageContract.ts | 2 +- .../admission/ReferenceAdmissionStorage.ts | 378 +++++++----- .../__tests/mongo/MongoCommitResponseProxy.ts | 16 +- .../mongo/MongoTransactionRunner.test.ts | 2 +- .../overlay/src/storage/AdmissionStorage.ts | 4 +- .../src/storage/mongo/MongoPayloadStore.ts | 572 ++++++++++-------- .../src/storage/mongo/MongoReadGuards.ts | 17 +- .../overlay/src/storage/mongo/MongoSchema.ts | 104 ++-- .../storage/mongo/MongoTransactionRunner.ts | 168 +++-- 10 files changed, 742 insertions(+), 529 deletions(-) diff --git a/packages/overlays/overlay/src/__tests/AdmissionStorage.test.ts b/packages/overlays/overlay/src/__tests/AdmissionStorage.test.ts index 36564abbb..3f7d299d7 100644 --- a/packages/overlays/overlay/src/__tests/AdmissionStorage.test.ts +++ b/packages/overlays/overlay/src/__tests/AdmissionStorage.test.ts @@ -1,3 +1,11 @@ +import { getAdmissionStorage } from '../storage/AdmissionStorage.js' import { referenceAdmissionStorageContract } from './admission/AdmissionStorageContract.js' referenceAdmissionStorageContract() + +describe('AdmissionStorage provider detection', () => { + test('does not treat a missing provider as admission storage', () => { + expect(getAdmissionStorage(undefined)).toBeUndefined() + expect(getAdmissionStorage({ admission: null })).toBeUndefined() + }) +}) diff --git a/packages/overlays/overlay/src/__tests/admission/AdmissionStorageContract.ts b/packages/overlays/overlay/src/__tests/admission/AdmissionStorageContract.ts index 320f28e84..ea4d75c5e 100644 --- a/packages/overlays/overlay/src/__tests/admission/AdmissionStorageContract.ts +++ b/packages/overlays/overlay/src/__tests/admission/AdmissionStorageContract.ts @@ -15,7 +15,7 @@ import { type AdmissionStorageContractHarness } from './ReferenceAdmissionStorage.js' -const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T +const clone = (value: T): T => structuredClone(value) const hash = (character: string): string => character.repeat(64) const history = { chainEpoch: '7', topicHistoryGeneration: '3' } diff --git a/packages/overlays/overlay/src/__tests/admission/ReferenceAdmissionStorage.ts b/packages/overlays/overlay/src/__tests/admission/ReferenceAdmissionStorage.ts index 37142b9ce..50533df40 100644 --- a/packages/overlays/overlay/src/__tests/admission/ReferenceAdmissionStorage.ts +++ b/packages/overlays/overlay/src/__tests/admission/ReferenceAdmissionStorage.ts @@ -120,7 +120,7 @@ export interface ReferenceAdmissionStorageOptions { projector?: unknown } -const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T +const clone = (value: T): T => structuredClone(value) const scopeKey = (scope: StorageScope): string => JSON.stringify([scope.network, scope.genesisHash, scope.nodeId]) @@ -158,10 +158,9 @@ const sameFence = (left: HistoryFence | undefined, right: HistoryFence): boolean left?.chainEpoch === right.chainEpoch && left.topicHistoryGeneration === right.topicHistoryGeneration const samePayload = (left: AdmissionPayloadRef | undefined, right: AdmissionPayloadRef): boolean => - left !== undefined && - left.digest === right.digest && - left.byteLength === right.byteLength && - left.kind === right.kind + left?.digest === right.digest && + left?.byteLength === right.byteLength && + left?.kind === right.kind const isHash = (value: string): boolean => /^[0-9a-f]{64}$/.test(value) const isUint64 = (value: string): boolean => { try { @@ -329,9 +328,24 @@ export class ReferenceAdmissionStorage implements AdmissionStorage { next: ReferenceState, plan: AdmissionCommit ): Extract | undefined { + if (this.hasInvalidPlanShape(next, plan) || !this.isSupportedPlan(plan, next)) + return { state: 'rejected', code: 'invalid-plan' } + const references = this.references(plan) + if ( + references === undefined || + references.some(ref => !samePayload(next.payloads.get(ref.digest), ref)) + ) { + return { state: 'rejected', code: 'payload-not-ready' } + } + const conflict = this.decisionPredicateConflict(next, plan) + if (conflict !== undefined) return conflict + return this.publishPlan(next, plan, references) + } + + private hasInvalidPlanShape(next: ReferenceState, plan: AdmissionCommit): boolean { const topics = new Set(plan.identity.topics.map(item => item.topic)) const decisionTopics = new Set(plan.decisions.map(decision => decision.topic)) - if ( + return ( topics.size !== plan.identity.topics.length || plan.decisions.length !== topics.size || decisionTopics.size !== plan.decisions.length || @@ -339,111 +353,135 @@ export class ReferenceAdmissionStorage implements AdmissionStorage { plan.decisions.some(decision => !topics.has(decision.topic)) || new Set(plan.outbox.map(intent => intent.eventId)).size !== plan.outbox.length || plan.outbox.some(intent => next.outbox.has(outboxKey(plan.identity.scope, intent.eventId))) - ) { - return { state: 'rejected', code: 'invalid-plan' } + ) + } + + private decisionPredicateConflict( + next: ReferenceState, + plan: AdmissionCommit + ): Extract | undefined { + for (const decision of plan.decisions) { + const conflict = this.singleDecisionConflict(next, plan, decision) + if (conflict !== undefined) return conflict } - if (!this.isSupportedPlan(plan, next)) return { state: 'rejected', code: 'invalid-plan' } + return undefined + } - const references = this.references(plan) + private singleDecisionConflict( + next: ReferenceState, + plan: AdmissionCommit, + decision: AdmissionTopicDecision + ): Extract | undefined { if ( - references === undefined || - references.some(ref => !samePayload(next.payloads.get(ref.digest), ref)) + !sameFence( + next.fences.get(topicKey(plan.identity.scope, decision.topic)), + decision.expectedHistory + ) ) { - return { state: 'rejected', code: 'payload-not-ready' } + return { state: 'rejected', code: 'read-conflict' } } - - for (const decision of plan.decisions) { - if ( - !sameFence( - next.fences.get(topicKey(plan.identity.scope, decision.topic)), - decision.expectedHistory - ) - ) { - return { state: 'rejected', code: 'read-conflict' } - } - if ( - decision.reads.some( - read => - (next.reads.get(readKey(plan.identity.scope, decision.topic, read.key)) ?? null) !== - read.expectedVersion - ) - ) { - return { state: 'rejected', code: 'read-conflict' } - } - if ( - decision.spends.some( - spend => - next.outputs.get(outputKey(plan.identity.scope, decision.topic, spend.outpoint)) - ?.version !== spend.expectedVersion || - next.outputs.get(outputKey(plan.identity.scope, decision.topic, spend.outpoint)) - ?.spentBy !== undefined - ) - ) { - return { state: 'rejected', code: 'spend-conflict' } - } - if ( - decision.historyUpdate?.handoff !== undefined && - !this.canHandoff( - next, - plan.identity.scope, - decision.topic, - decision.expectedHistory, - decision.historyUpdate.handoff - ) - ) { - return { state: 'rejected', code: 'read-conflict' } - } + if ( + decision.reads.some( + read => + (next.reads.get(readKey(plan.identity.scope, decision.topic, read.key)) ?? null) !== + read.expectedVersion + ) + ) { + return { state: 'rejected', code: 'read-conflict' } + } + if ( + decision.spends.some( + spend => + next.outputs.get(outputKey(plan.identity.scope, decision.topic, spend.outpoint)) + ?.version !== spend.expectedVersion || + next.outputs.get(outputKey(plan.identity.scope, decision.topic, spend.outpoint)) + ?.spentBy !== undefined + ) + ) { + return { state: 'rejected', code: 'spend-conflict' } } + if ( + decision.historyUpdate?.handoff !== undefined && + !this.canHandoff( + next, + plan.identity.scope, + decision.topic, + decision.expectedHistory, + decision.historyUpdate.handoff + ) + ) { + return { state: 'rejected', code: 'read-conflict' } + } + return undefined + } + private publishPlan( + next: ReferenceState, + plan: AdmissionCommit, + references: AdmissionPayloadRef[] + ): Extract | undefined { for (const reference of references) next.pins.add(reference.digest) for (const decision of plan.decisions) { - for (const spend of decision.spends) { - const stored = next.outputs.get( - outputKey(plan.identity.scope, decision.topic, spend.outpoint) - ) - if (stored !== undefined) stored.spentBy = spend.spender - } - for (const eviction of decision.evictions) - next.outputs.delete(outputKey(plan.identity.scope, decision.topic, eviction)) - for (const output of decision.outputs) { - const key = outputKey(plan.identity.scope, decision.topic, output) - if (next.outputs.has(key)) return { state: 'rejected', code: 'invalid-plan' } - next.outputs.set(key, { version: '1', topic: decision.topic, output: clone(output) }) - } - for (const edge of decision.edges) - next.edges.add(edgeKey(plan.identity.scope, decision.topic, edge.source, edge.consumer)) - next.applied.set( - appliedKey(plan.identity.scope, decision.topic, decision.applied.txid), - clone(decision.applied) - ) - if (decision.historyUpdate !== undefined) { - next.fences.set(topicKey(plan.identity.scope, decision.topic), { - chainEpoch: decision.expectedHistory.chainEpoch, - topicHistoryGeneration: decision.historyUpdate.nextTopicHistoryGeneration - }) - if (decision.historyUpdate.handoff !== undefined) { - const lease = next.leases.get(leaseKey(decision.historyUpdate.handoff.expected)) - if (lease !== undefined) { - lease.topicHistoryGeneration = decision.historyUpdate.nextTopicHistoryGeneration - } - next.handoffs.set( - topicKey(plan.identity.scope, decision.topic), - decision.historyUpdate.handoff.checkpoint - ) - } - next.historyUpdates.set(topicKey(plan.identity.scope, decision.topic), { - affectedFromHeight: decision.historyUpdate.affectedFromHeight, - ...(decision.historyUpdate.handoff === undefined - ? {} - : { checkpoint: decision.historyUpdate.handoff.checkpoint }) - }) - } + const invalid = this.publishDecision(next, plan, decision) + if (invalid !== undefined) return invalid } for (const intent of plan.outbox) next.outbox.set(outboxKey(plan.identity.scope, intent.eventId), clone(intent)) return undefined } + private publishDecision( + next: ReferenceState, + plan: AdmissionCommit, + decision: AdmissionTopicDecision + ): Extract | undefined { + for (const spend of decision.spends) { + const stored = next.outputs.get( + outputKey(plan.identity.scope, decision.topic, spend.outpoint) + ) + if (stored !== undefined) stored.spentBy = spend.spender + } + for (const eviction of decision.evictions) + next.outputs.delete(outputKey(plan.identity.scope, decision.topic, eviction)) + for (const output of decision.outputs) { + const key = outputKey(plan.identity.scope, decision.topic, output) + if (next.outputs.has(key)) return { state: 'rejected', code: 'invalid-plan' } + next.outputs.set(key, { version: '1', topic: decision.topic, output: clone(output) }) + } + for (const edge of decision.edges) + next.edges.add(edgeKey(plan.identity.scope, decision.topic, edge.source, edge.consumer)) + next.applied.set( + appliedKey(plan.identity.scope, decision.topic, decision.applied.txid), + clone(decision.applied) + ) + if (decision.historyUpdate !== undefined) this.publishHistoryUpdate(next, plan, decision) + return undefined + } + + private publishHistoryUpdate( + next: ReferenceState, + plan: AdmissionCommit, + decision: AdmissionTopicDecision + ): void { + const historyUpdate = decision.historyUpdate + if (historyUpdate === undefined) return + next.fences.set(topicKey(plan.identity.scope, decision.topic), { + chainEpoch: decision.expectedHistory.chainEpoch, + topicHistoryGeneration: historyUpdate.nextTopicHistoryGeneration + }) + if (historyUpdate.handoff !== undefined) { + const lease = next.leases.get(leaseKey(historyUpdate.handoff.expected)) + if (lease !== undefined) { + lease.topicHistoryGeneration = historyUpdate.nextTopicHistoryGeneration + } + next.handoffs.set(topicKey(plan.identity.scope, decision.topic), historyUpdate.handoff.checkpoint) + } + next.historyUpdates.set(topicKey(plan.identity.scope, decision.topic), { + affectedFromHeight: historyUpdate.affectedFromHeight, + ...(historyUpdate.handoff === undefined ? {} : { checkpoint: historyUpdate.handoff.checkpoint }) + }) + } + private references(plan: AdmissionCommit): AdmissionPayloadRef[] | undefined { const references = [...plan.payloads] for (const decision of plan.decisions) { @@ -470,77 +508,85 @@ export class ReferenceAdmissionStorage implements AdmissionStorage { ) return false if (!this.isBoundSteak(statePlan)) return false - for (const decision of statePlan.decisions) { - if ( - !isUint64(decision.expectedHistory.chainEpoch) || - !isUint64(decision.expectedHistory.topicHistoryGeneration) - ) - return false - if ( - decision.spends.some( - spend => !isWireOutpoint(spend.outpoint) || spend.spender !== statePlan.identity.txid - ) - ) - return false - if ( - decision.evictions.some( - eviction => - !isWireOutpoint(eviction) || - !state.outputs.has(outputKey(statePlan.identity.scope, decision.topic, eviction)) - ) - ) - return false - if ( - decision.outputs.some(output => { - if (output.txid !== statePlan.identity.txid || !isWireOutpoint(output)) return true - if ( - ![output.satoshis, output.score, output.script.offset, output.script.byteLength].every( - isUint64 - ) - ) - return true - if (!isHash(output.script.payload.digest) || !isUint64(output.script.payload.byteLength)) - return true - return ( - parseStorageUint64(output.script.offset) + - parseStorageUint64(output.script.byteLength) > - parseStorageUint64(output.script.payload.byteLength) - ) - }) - ) - return false - if ( - decision.edges.some(edge => !isWireOutpoint(edge.source) || !isWireOutpoint(edge.consumer)) - ) - return false - const applied = decision.applied - if ( - applied.txid !== statePlan.identity.txid || - !isHash(applied.txid) || - state.applied.has(appliedKey(statePlan.identity.scope, decision.topic, applied.txid)) + return statePlan.decisions.every(decision => this.decisionIsSupported(statePlan, state, decision)) + } + + private decisionIsSupported( + statePlan: AdmissionCommit, + state: ReferenceState, + decision: AdmissionTopicDecision + ): boolean { + if ( + !isUint64(decision.expectedHistory.chainEpoch) || + !isUint64(decision.expectedHistory.topicHistoryGeneration) + ) + return false + if ( + decision.spends.some( + spend => !isWireOutpoint(spend.outpoint) || spend.spender !== statePlan.identity.txid ) - return false - if ( - (applied.firstSeenHeight !== undefined && !isUint64(applied.firstSeenHeight)) || - (applied.proof !== undefined && - (!isHash(applied.proof.digest) || !isUint64(applied.proof.byteLength))) || - (applied.block !== undefined && - (![applied.block.height, applied.block.index].every(isUint64) || - !isHash(applied.block.hash) || - !isHash(applied.block.merkleRoot))) + ) + return false + if ( + decision.evictions.some( + eviction => + !isWireOutpoint(eviction) || + !state.outputs.has(outputKey(statePlan.identity.scope, decision.topic, eviction)) ) - return false - if (decision.historyUpdate !== undefined) { - if ( - !isUint64(decision.historyUpdate.nextTopicHistoryGeneration) || - !isUint64(decision.historyUpdate.affectedFromHeight) || - parseStorageUint64(decision.historyUpdate.nextTopicHistoryGeneration) <= - parseStorageUint64(decision.expectedHistory.topicHistoryGeneration) - ) - return false - } - } - return true + ) + return false + if (decision.outputs.some(output => this.outputViolatesPlan(statePlan, output))) return false + if (decision.edges.some(edge => !isWireOutpoint(edge.source) || !isWireOutpoint(edge.consumer))) + return false + if (this.appliedViolatesPlan(statePlan, state, decision)) return false + return !this.historyUpdateViolatesPlan(decision) + } + + private outputViolatesPlan(statePlan: AdmissionCommit, output: AdmissionOutput): boolean { + if (output.txid !== statePlan.identity.txid || !isWireOutpoint(output)) return true + if ( + ![output.satoshis, output.score, output.script.offset, output.script.byteLength].every(isUint64) + ) + return true + if (!isHash(output.script.payload.digest) || !isUint64(output.script.payload.byteLength)) + return true + return ( + parseStorageUint64(output.script.offset) + parseStorageUint64(output.script.byteLength) > + parseStorageUint64(output.script.payload.byteLength) + ) + } + + private appliedViolatesPlan( + statePlan: AdmissionCommit, + state: ReferenceState, + decision: AdmissionTopicDecision + ): boolean { + const applied = decision.applied + if ( + applied.txid !== statePlan.identity.txid || + !isHash(applied.txid) || + state.applied.has(appliedKey(statePlan.identity.scope, decision.topic, applied.txid)) + ) + return true + return ( + (applied.firstSeenHeight !== undefined && !isUint64(applied.firstSeenHeight)) || + (applied.proof !== undefined && + (!isHash(applied.proof.digest) || !isUint64(applied.proof.byteLength))) || + (applied.block !== undefined && + (![applied.block.height, applied.block.index].every(isUint64) || + !isHash(applied.block.hash) || + !isHash(applied.block.merkleRoot))) + ) + } + + private historyUpdateViolatesPlan(decision: AdmissionTopicDecision): boolean { + if (decision.historyUpdate === undefined) return false + return ( + !isUint64(decision.historyUpdate.nextTopicHistoryGeneration) || + !isUint64(decision.historyUpdate.affectedFromHeight) || + parseStorageUint64(decision.historyUpdate.nextTopicHistoryGeneration) <= + parseStorageUint64(decision.expectedHistory.topicHistoryGeneration) + ) } private isBoundSteak(plan: AdmissionCommit): boolean { @@ -674,7 +720,7 @@ export class ReferenceAdmissionHarness implements AdmissionStorageContractHarnes })), leases: [...state.leases.values()].map(lease => clone(lease)), outputs: [...state.outputs].map(([key, output]) => ({ key, ...clone(output) })), - edges: [...state.edges].sort(), + edges: [...state.edges].sort((left, right) => left.localeCompare(right, 'en')), applied: [...state.applied].map(([key, record]) => ({ key, record: clone(record) })), outbox: [...state.outbox.keys()].map(key => { const [network, genesisHash, nodeId, eventId] = JSON.parse(key) as [ diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseProxy.ts b/packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseProxy.ts index 0e0e62bc1..aec88fbf9 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseProxy.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoCommitResponseProxy.ts @@ -5,6 +5,14 @@ const maxFrameBytes = 4 * 1024 * 1024 const maxBufferedBytes = maxFrameBytes * 2 const opMessage = 2013 +function encodeMongoTxnNumber(value: unknown): string { + if (typeof value === 'number' && Number.isSafeInteger(value)) return value.toString(10) + if (typeof value === 'bigint') return value.toString(10) + if (typeof value === 'string' && /^-?\d+$/.test(value)) return value + if (BSON.Long.isLong(value)) return value.toString() + throw new Error('Mongo commit lacks session identity') +} + export interface CapturedMongoCommit { readonly requestId: number readonly lsid: string @@ -165,16 +173,16 @@ export class MongoCommitResponseProxy { private observeRequest(connection: Connection, message: Buffer): void { const header = this.header(message) const command = this.command(message) - if (header === undefined || command === undefined || command.commitTransaction !== 1) return - const lsid = command.lsid - const txnNumber = command.txnNumber + if (header === undefined || command?.commitTransaction !== 1) return + const lsid = command?.lsid + const txnNumber = command?.txnNumber if (lsid === undefined || txnNumber === undefined) throw new Error('Mongo commit lacks session identity') connection.commitRequests.add(header.requestId) this.commits.push({ requestId: header.requestId, lsid: JSON.stringify(lsid), - txnNumber: String(txnNumber) + txnNumber: encodeMongoTxnNumber(txnNumber) }) } diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts index 9519d4c85..39a8ddb62 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts @@ -123,7 +123,7 @@ describe('Mongo transaction boundary on three data-bearing WiredTiger members', expect(result).toEqual({ state: 'committed', receipt: input.receipt }) expect(bodies).toBe(1) const commits = commands.slice(start).filter(event => event.commandName === 'commitTransaction') - expect(commits.length).toBe(3) + expect(commits).toHaveLength(3) expect(new Set(commits.map(event => `${String(event.command.lsid.id)}:${String(event.command.txnNumber)}`)).size).toBe(1) }, 15000) diff --git a/packages/overlays/overlay/src/storage/AdmissionStorage.ts b/packages/overlays/overlay/src/storage/AdmissionStorage.ts index 6d861fd8f..a7c34959a 100644 --- a/packages/overlays/overlay/src/storage/AdmissionStorage.ts +++ b/packages/overlays/overlay/src/storage/AdmissionStorage.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto' import type { HistoryRevisionHandoff } from './RecoveryContract.js' /** Canonical unsigned decimal, 0..2^64-1. Never round through a JS number. */ -export type StorageUint64 = string +export type StorageUint64 = string & { readonly __brand: unique symbol } export interface StorageScope { network: string @@ -194,7 +194,7 @@ export function isReplaySafeProjection(projection: unknown): projection is Repla } export function parseStorageUint64(value: string): bigint { - if (typeof value !== 'string' || !/^(0|[1-9][0-9]{0,19})$/.test(value)) + if (typeof value !== 'string' || !/^(0|[1-9]\d{0,19})$/.test(value)) throw new Error('Invalid storage uint64') const result = BigInt(value) if (result > BigInt('18446744073709551615')) throw new Error('Invalid storage uint64') diff --git a/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts b/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts index ab899ecd2..30e2bf9d9 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts @@ -130,6 +130,14 @@ interface ReferenceDocument extends Document { expiresAt?: Date } +interface PayloadStream { + hash: ReturnType + length: bigint + inline: Uint8Array[] | undefined + upload: ReturnType | undefined + fileId: ObjectId | undefined +} + const MAX_UINT64 = (BigInt(1) << BigInt(64)) - BigInt(1) const HEX_256 = /^[0-9a-f]{64}$/ @@ -193,100 +201,7 @@ export class MongoPayloadStore { const payloads = this.db.collection(MongoCollectionNames.payloads) const reserved = await this.reserveUpload(payloads, payloadId, input) if (reserved.ready) return this.refFor(input, reserved.record) - const ownerId = reserved.record.ownerId - const fence = reserved.record.fencingToken - - let fileId: ObjectId | undefined - try { - const streamed = await this.streamInput(input, payloadId, ownerId, fence) - this.throwIfAborted(input.signal) - fileId = streamed.fileId - if (streamed.digest !== input.digest || streamed.length !== BigInt(input.byteLength)) - throw new Error('Mongo payload digest or declared length mismatch') - if ( - input.kind === 'raw-transaction' && - input.txid !== undefined && - input.txid !== streamed.txid - ) - throw new Error('Mongo raw transaction id does not match bytes') - const binary = - streamed.inline === undefined ? undefined : new Binary(Buffer.concat(streamed.inline)) - if (binary !== undefined && !this.fitsInline(payloadId, input, binary, this.now())) - throw new Error('Mongo payload inline BSON document exceeds safety ceiling') - if (fileId !== undefined) { - const staged = await payloads.updateOne( - { _id: payloadId, state: 'uploading', ownerId, guard: ownerId }, - { $set: { fileId, updatedAt: this.now() } }, - { writeConcern: this.writeConcern } - ) - if (staged.matchedCount !== 1) - throw new Error('Mongo payload upload lease was fenced before staging') - await this.verifyGridFs(fileId, input, input.signal) - this.throwIfAborted(input.signal) - await this.options.hooks?.afterGridFsUploaded?.() - this.throwIfAborted(input.signal) - const published = await this.db.collection(`${MongoGridFsBucketName}.files`).updateOne( - { - _id: fileId, - 'metadata.ownerId': ownerId, - 'metadata.fencingToken': fence, - 'metadata.state': 'staged' - }, - { $set: { 'metadata.state': 'published' } }, - { writeConcern: this.writeConcern } - ) - if (published.matchedCount !== 1) throw new Error('Mongo GridFS staged file was lost') - await this.options.hooks?.afterGridFsPublished?.() - } - await this.options.hooks?.beforeReadyCas?.() - this.throwIfAborted(input.signal) - const ready = await payloads.findOneAndUpdate( - { - _id: payloadId, - state: 'uploading', - ownerId, - guard: ownerId, - $expr: { $gt: ['$leaseUntil', '$$NOW'] } - }, - { - $set: { - state: 'ready', - updatedAt: this.now(), - ...(fileId === undefined ? { inlineData: binary } : { fileId }) - }, - $unset: { uploadId: '' } - }, - { returnDocument: 'after', writeConcern: this.writeConcern } - ) - if (ready === null) - throw new Error('Mongo payload upload lease was fenced before publication') - return this.refFor(input, ready) - } catch (error) { - // Never remove a file by payload identity: a newer fenced owner may have published it. - if (fileId !== undefined) { - const published = await this.db.collection(`${MongoGridFsBucketName}.files`).findOne({ - _id: fileId, - 'metadata.ownerId': ownerId, - 'metadata.fencingToken': fence, - 'metadata.state': 'published' - }) - if (published !== null) throw error - await this.retireOwnedFile(fileId, ownerId) - } - await payloads.updateOne( - { _id: payloadId, state: 'uploading', ownerId, guard: ownerId }, - { - $set: { - state: 'deleted', - updatedAt: this.now(), - ...(fileId === undefined ? {} : { retiredFileId: fileId }) - }, - $unset: { inlineData: '', fileId: '' } - }, - { writeConcern: this.writeConcern } - ) - throw error - } + return await this.completeUpload(payloads, input, payloadId, reserved.record) } /** Must be called in the same transaction that creates the referenced owner record. */ @@ -503,61 +418,191 @@ export class MongoPayloadStore { $expr: { $lte: ['$leaseUntil', '$$NOW'] } }) .toArray() - for (const record of stale) { - if (record.fileId !== undefined) { - try { - await this.verifyGridFs(record.fileId, { - digest: record.digest, - byteLength: BigInt(record.byteLength).toString() - }) - await this.db.collection(`${MongoGridFsBucketName}.files`).updateOne( - { - _id: record.fileId, - 'metadata.ownerId': record.ownerId, - 'metadata.fencingToken': record.fencingToken - }, - { $set: { 'metadata.state': 'published' } }, - { writeConcern: this.writeConcern } - ) - const recovered = await payloads.updateOne( - { _id: record._id, state: 'uploading', ownerId: record.ownerId, guard: record.guard }, - { $set: { state: 'ready', updatedAt: this.now() }, $unset: { uploadId: '' } }, - { writeConcern: this.writeConcern } - ) - if (recovered.modifiedCount === 1) continue - } catch { - /* an incomplete upload is retired below */ - } - } - const current = await payloads.findOne({ _id: record._id }) - if ( - current === null || - current.state !== 'uploading' || - current.ownerId !== record.ownerId || - current.guard !== record.guard - ) { - continue - } - if (record.fileId !== undefined) - await this.retireOwnedFile(record.fileId, record.ownerId, record.fencingToken) - await payloads.updateOne( - { _id: record._id, state: 'uploading', ownerId: record.ownerId, guard: record.guard }, + for (const record of stale) await this.recoverStaleUpload(payloads, record) + const deleting = await payloads + .find({ network: this.scope.network, genesisHash: this.scope.genesisHash, state: 'deleting' }) + .toArray() + for (const record of deleting) + await this.finishGarbage({ kind: record.kind, digest: record.digest }) + } + + private async completeUpload( + payloads: Collection, + input: MongoPayloadInput, + payloadId: string, + reserved: PayloadDocument + ): Promise { + const ownerId = reserved.ownerId + const fence = reserved.fencingToken + let fileId: ObjectId | undefined + try { + const streamed = await this.streamInput(input, payloadId, ownerId, fence) + this.throwIfAborted(input.signal) + fileId = streamed.fileId + this.assertStreamedPayload(input, streamed) + const binary = + streamed.inline === undefined ? undefined : new Binary(Buffer.concat(streamed.inline)) + if (binary !== undefined && !this.fitsInline(payloadId, input, binary, this.now())) + throw new Error('Mongo payload inline BSON document exceeds safety ceiling') + if (fileId !== undefined) + await this.stageGridFsUpload(payloads, input, payloadId, ownerId, fence, fileId) + await this.options.hooks?.beforeReadyCas?.() + this.throwIfAborted(input.signal) + const ready = await payloads.findOneAndUpdate( + { + _id: payloadId, + state: 'uploading', + ownerId, + guard: ownerId, + $expr: { $gt: ['$leaseUntil', '$$NOW'] } + }, { $set: { - state: 'deleted', + state: 'ready', updatedAt: this.now(), - ...(record.fileId === undefined ? {} : { retiredFileId: record.fileId }) + ...(fileId === undefined ? { inlineData: binary } : { fileId }) }, - $unset: { inlineData: '', fileId: '' } + $unset: { uploadId: '' } }, - { writeConcern: this.writeConcern } + { returnDocument: 'after', writeConcern: this.writeConcern } ) + if (ready === null) + throw new Error('Mongo payload upload lease was fenced before publication') + return this.refFor(input, ready) + } catch (error) { + await this.abandonUpload(payloads, payloadId, ownerId, fence, fileId, error) + throw error } - const deleting = await payloads - .find({ network: this.scope.network, genesisHash: this.scope.genesisHash, state: 'deleting' }) - .toArray() - for (const record of deleting) - await this.finishGarbage({ kind: record.kind, digest: record.digest }) + } + + private assertStreamedPayload( + input: MongoPayloadInput, + streamed: { digest: string; length: bigint; txid?: string } + ): void { + if (streamed.digest !== input.digest || streamed.length !== BigInt(input.byteLength)) + throw new Error('Mongo payload digest or declared length mismatch') + if (input.kind === 'raw-transaction' && input.txid !== undefined && input.txid !== streamed.txid) + throw new Error('Mongo raw transaction id does not match bytes') + } + + private async stageGridFsUpload( + payloads: Collection, + input: MongoPayloadInput, + payloadId: string, + ownerId: string, + fence: string, + fileId: ObjectId + ): Promise { + const staged = await payloads.updateOne( + { _id: payloadId, state: 'uploading', ownerId, guard: ownerId }, + { $set: { fileId, updatedAt: this.now() } }, + { writeConcern: this.writeConcern } + ) + if (staged.matchedCount !== 1) + throw new Error('Mongo payload upload lease was fenced before staging') + await this.verifyGridFs(fileId, input, input.signal) + this.throwIfAborted(input.signal) + await this.options.hooks?.afterGridFsUploaded?.() + this.throwIfAborted(input.signal) + const published = await this.db.collection(`${MongoGridFsBucketName}.files`).updateOne( + { + _id: fileId, + 'metadata.ownerId': ownerId, + 'metadata.fencingToken': fence, + 'metadata.state': 'staged' + }, + { $set: { 'metadata.state': 'published' } }, + { writeConcern: this.writeConcern } + ) + if (published.matchedCount !== 1) throw new Error('Mongo GridFS staged file was lost') + await this.options.hooks?.afterGridFsPublished?.() + } + + private async abandonUpload( + payloads: Collection, + payloadId: string, + ownerId: string, + fence: string, + fileId: ObjectId | undefined, + error: unknown + ): Promise { + // Never remove a file by payload identity: a newer fenced owner may have published it. + if (fileId !== undefined) { + const published = await this.db.collection(`${MongoGridFsBucketName}.files`).findOne({ + _id: fileId, + 'metadata.ownerId': ownerId, + 'metadata.fencingToken': fence, + 'metadata.state': 'published' + }) + if (published !== null) throw error + await this.retireOwnedFile(fileId, ownerId) + } + await payloads.updateOne( + { _id: payloadId, state: 'uploading', ownerId, guard: ownerId }, + { + $set: { + state: 'deleted', + updatedAt: this.now(), + ...(fileId === undefined ? {} : { retiredFileId: fileId }) + }, + $unset: { inlineData: '', fileId: '' } + }, + { writeConcern: this.writeConcern } + ) + } + + private async recoverStaleUpload( + payloads: Collection, + record: PayloadDocument + ): Promise { + if (record.fileId !== undefined) { + try { + await this.verifyGridFs(record.fileId, { + digest: record.digest, + byteLength: BigInt(record.byteLength).toString() + }) + await this.db.collection(`${MongoGridFsBucketName}.files`).updateOne( + { + _id: record.fileId, + 'metadata.ownerId': record.ownerId, + 'metadata.fencingToken': record.fencingToken + }, + { $set: { 'metadata.state': 'published' } }, + { writeConcern: this.writeConcern } + ) + const recovered = await payloads.updateOne( + { _id: record._id, state: 'uploading', ownerId: record.ownerId, guard: record.guard }, + { $set: { state: 'ready', updatedAt: this.now() }, $unset: { uploadId: '' } }, + { writeConcern: this.writeConcern } + ) + if (recovered.modifiedCount === 1) return + } catch { + /* an incomplete upload is retired below */ + } + } + const current = await payloads.findOne({ _id: record._id }) + if ( + current === null || + current.state !== 'uploading' || + current.ownerId !== record.ownerId || + current.guard !== record.guard + ) { + return + } + if (record.fileId !== undefined) + await this.retireOwnedFile(record.fileId, record.ownerId, record.fencingToken) + await payloads.updateOne( + { _id: record._id, state: 'uploading', ownerId: record.ownerId, guard: record.guard }, + { + $set: { + state: 'deleted', + updatedAt: this.now(), + ...(record.fileId === undefined ? {} : { retiredFileId: record.fileId }) + }, + $unset: { inlineData: '', fileId: '' } + }, + { writeConcern: this.writeConcern } + ) } private async reserveUpload( @@ -570,55 +615,8 @@ export class MongoPayloadStore { this.throwIfAborted(input.signal) const observed = await payloads.findOne({ _id: payloadId }) if (observed?.state === 'ready') return { ready: true, record: observed } - const ownerId = randomUUID() - const fence = encodeMongoUint64( - (BigInt(observed?.fencingToken ?? '00000000000000000000') + BigInt(1)).toString() - ) - try { - const claimed = await payloads.findOneAndUpdate( - observed === null - ? { _id: payloadId, state: { $exists: false } } - : { - _id: payloadId, - state: observed.state, - guard: observed.guard, - fencingToken: observed.fencingToken, - $or: [{ state: 'deleted' }, { $expr: { $lte: ['$leaseUntil', '$$NOW'] } }] - }, - [ - { - $set: { - schemaVersion: { $ifNull: ['$schemaVersion', 1] }, - network: { $ifNull: ['$network', this.scope.network] }, - genesisHash: { $ifNull: ['$genesisHash', this.scope.genesisHash] }, - kind: { $ifNull: ['$kind', input.kind] }, - digest: { $ifNull: ['$digest', input.digest] }, - byteLength: { $ifNull: ['$byteLength', encodeMongoUint64(input.byteLength)] }, - createdAt: { $ifNull: ['$createdAt', '$$NOW'] }, - state: 'uploading', - guard: ownerId, - ownerNodeId: this.scope.nodeId, - ownerId, - fencingToken: fence, - leaseUntil: { - $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.uploadLeaseMs } - }, - updatedAt: '$$NOW', - uploadId: ownerId - } - }, - { $unset: ['inlineData', 'fileId'] } - ], - { upsert: observed === null, returnDocument: 'after', writeConcern: this.writeConcern } - ) - if (claimed !== null && claimed.ownerId === ownerId && claimed.guard === ownerId) { - if (observed?.fileId !== undefined) - await this.retireOwnedFile(observed.fileId, observed.ownerId, observed.fencingToken) - return { ready: false, record: claimed } - } - } catch (error) { - if (!duplicateKey(error)) throw error - } + const claimed = await this.tryClaimUpload(payloads, payloadId, input, observed) + if (claimed !== undefined) return claimed const ready = await payloads.findOne({ _id: payloadId, state: 'ready' }) if (ready !== null) return { ready: true, record: ready } if (Date.now() > deadline) throw new Error('Mongo payload upload reservation was lost') @@ -626,6 +624,64 @@ export class MongoPayloadStore { } } + private async tryClaimUpload( + payloads: Collection, + payloadId: string, + input: MongoPayloadInput, + observed: PayloadDocument | null + ): Promise<{ ready: false; record: PayloadDocument } | undefined> { + const ownerId = randomUUID() + const fence = encodeMongoUint64( + (BigInt(observed?.fencingToken ?? '00000000000000000000') + BigInt(1)).toString() + ) + try { + const claimed = await payloads.findOneAndUpdate( + observed === null + ? { _id: payloadId, state: { $exists: false } } + : { + _id: payloadId, + state: observed.state, + guard: observed.guard, + fencingToken: observed.fencingToken, + $or: [{ state: 'deleted' }, { $expr: { $lte: ['$leaseUntil', '$$NOW'] } }] + }, + [ + { + $set: { + schemaVersion: { $ifNull: ['$schemaVersion', 1] }, + network: { $ifNull: ['$network', this.scope.network] }, + genesisHash: { $ifNull: ['$genesisHash', this.scope.genesisHash] }, + kind: { $ifNull: ['$kind', input.kind] }, + digest: { $ifNull: ['$digest', input.digest] }, + byteLength: { $ifNull: ['$byteLength', encodeMongoUint64(input.byteLength)] }, + createdAt: { $ifNull: ['$createdAt', '$$NOW'] }, + state: 'uploading', + guard: ownerId, + ownerNodeId: this.scope.nodeId, + ownerId, + fencingToken: fence, + leaseUntil: { + $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.uploadLeaseMs } + }, + updatedAt: '$$NOW', + uploadId: ownerId + } + }, + { $unset: ['inlineData', 'fileId'] } + ], + { upsert: observed === null, returnDocument: 'after', writeConcern: this.writeConcern } + ) + if (claimed !== null && claimed.ownerId === ownerId && claimed.guard === ownerId) { + if (observed?.fileId !== undefined) + await this.retireOwnedFile(observed.fileId, observed.ownerId, observed.fencingToken) + return { ready: false, record: claimed } + } + } catch (error) { + if (!duplicateKey(error)) throw error + } + return undefined + } + private async streamInput( input: MongoPayloadInput, payloadId: string, @@ -638,74 +694,112 @@ export class MongoPayloadStore { inline?: Uint8Array[] fileId?: ObjectId }> { - const hash = createHash('sha256') - let length = BigInt(0) - let inline: Uint8Array[] | undefined = [] - let upload: ReturnType | undefined - let fileId: ObjectId | undefined + const stream: PayloadStream = { + hash: createHash('sha256'), + length: BigInt(0), + inline: [], + upload: undefined, + fileId: undefined + } const iterator = input.bytes[Symbol.asyncIterator]() try { while (true) { const next = await this.nextWithAbort(iterator, input.signal) if (next.done) break - const chunk = next.value - if (input.signal?.aborted) - throw input.signal.reason ?? new Error('Mongo payload upload aborted') - if (!(chunk instanceof Uint8Array)) - throw new Error('Mongo payload stream yielded a non-byte chunk') - length += BigInt(chunk.byteLength) - if (length > BigInt(input.byteLength)) - throw new Error('Mongo payload stream exceeds declared length') - if (length > this.maxPayloadBytes) - throw new Error('Mongo payload stream exceeds configured byte bound') - hash.update(chunk) - if (inline !== undefined) { - inline.push(chunk) - if (length > BigInt(this.inlineCeiling)) { - fileId = new ObjectId() - upload = this.bucket.openUploadStreamWithId(fileId, payloadId, { - chunkSizeBytes: this.gridFsChunkBytes, - metadata: { - state: 'staged', - payloadId, - kind: input.kind, - digest: input.digest, - byteLength: encodeMongoUint64(input.byteLength), - ownerId, - fencingToken - } - }) - for (const buffered of inline) - if (!upload.write(Buffer.from(buffered))) - await this.awaitAbort(once(upload, 'drain'), input.signal) - inline = undefined - } - } else if (upload === undefined) throw new Error('Mongo payload stream writer missing') - else if (!upload.write(Buffer.from(chunk))) - await this.awaitAbort(once(upload, 'drain'), input.signal) - } - if (upload !== undefined) { - upload.end() - await this.awaitAbort(once(upload, 'finish'), input.signal) + await this.consumePayloadChunk(input, payloadId, ownerId, fencingToken, next.value, stream) } + await this.finishPayloadUpload(stream.upload, input.signal) } catch (error) { - upload?.destroy(error instanceof Error ? error : new Error('Mongo payload stream failed')) + stream.upload?.destroy( + error instanceof Error ? error : new Error('Mongo payload stream failed') + ) await iterator.return?.().catch(() => undefined) throw error } - const digest = hash.digest() + const digest = stream.hash.digest() const result: { digest: string length: bigint txid?: string inline?: Uint8Array[] fileId?: ObjectId - } = { digest: digest.toString('hex'), length, inline, fileId } + } = { + digest: digest.toString('hex'), + length: stream.length, + inline: stream.inline, + fileId: stream.fileId + } if (input.kind === 'raw-transaction') result.txid = createHash('sha256').update(digest).digest().reverse().toString('hex') return result } + private async consumePayloadChunk( + input: MongoPayloadInput, + payloadId: string, + ownerId: string, + fencingToken: string, + chunk: unknown, + stream: PayloadStream + ): Promise { + if (input.signal?.aborted) + throw input.signal.reason ?? new Error('Mongo payload upload aborted') + if (!(chunk instanceof Uint8Array)) + throw new Error('Mongo payload stream yielded a non-byte chunk') + stream.length += BigInt(chunk.byteLength) + if (stream.length > BigInt(input.byteLength)) + throw new Error('Mongo payload stream exceeds declared length') + if (stream.length > this.maxPayloadBytes) + throw new Error('Mongo payload stream exceeds configured byte bound') + stream.hash.update(chunk) + if (stream.inline !== undefined) { + stream.inline.push(chunk) + if (stream.length > BigInt(this.inlineCeiling)) + await this.spillInlineToGridFs(input, payloadId, ownerId, fencingToken, stream) + return + } + if (stream.upload === undefined) throw new Error('Mongo payload stream writer missing') + if (!stream.upload.write(Buffer.from(chunk))) + await this.awaitAbort(once(stream.upload, 'drain'), input.signal) + } + + private async spillInlineToGridFs( + input: MongoPayloadInput, + payloadId: string, + ownerId: string, + fencingToken: string, + stream: PayloadStream + ): Promise { + const buffered = stream.inline + if (buffered === undefined) throw new Error('Mongo payload stream writer missing') + stream.fileId = new ObjectId() + stream.upload = this.bucket.openUploadStreamWithId(stream.fileId, payloadId, { + chunkSizeBytes: this.gridFsChunkBytes, + metadata: { + state: 'staged', + payloadId, + kind: input.kind, + digest: input.digest, + byteLength: encodeMongoUint64(input.byteLength), + ownerId, + fencingToken + } + }) + stream.inline = undefined + for (const item of buffered) + if (!stream.upload.write(Buffer.from(item))) + await this.awaitAbort(once(stream.upload, 'drain'), input.signal) + } + + private async finishPayloadUpload( + upload: ReturnType | undefined, + signal?: AbortSignal + ): Promise { + if (upload === undefined) return + upload.end() + await this.awaitAbort(once(upload, 'finish'), signal) + } + private fitsInline( payloadId: string, input: MongoPayloadInput, @@ -884,6 +978,6 @@ export class MongoPayloadStore { } private isUint64(value: string): boolean { - return /^(0|[1-9][0-9]{0,19})$/.test(value) && BigInt(value) <= MAX_UINT64 + return /^(0|[1-9]\d{0,19})$/.test(value) && BigInt(value) <= MAX_UINT64 } } diff --git a/packages/overlays/overlay/src/storage/mongo/MongoReadGuards.ts b/packages/overlays/overlay/src/storage/mongo/MongoReadGuards.ts index 2b6f40d39..436e2a2bd 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoReadGuards.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoReadGuards.ts @@ -169,27 +169,12 @@ export class MongoReadGuards { value.length === 0 || Buffer.byteLength(value, 'utf8') > maximumBytes || value.includes('\u0000') || - !this.isWellFormed(value) + !value.isWellFormed() ) { throw new Error(`Invalid Mongo read guard ${label}`) } } - private isWellFormed(value: string): boolean { - for (let index = 0; index < value.length; index += 1) { - const unit = value.charCodeAt(index) - if (unit >= 0xd800 && unit <= 0xdbff) { - if (index + 1 >= value.length) return false - const next = value.charCodeAt(index + 1) - if (next < 0xdc00 || next > 0xdfff) return false - index += 1 - } else if (unit >= 0xdc00 && unit <= 0xdfff) { - return false - } - } - return true - } - private assertTransaction(session: ClientSession): void { if (!session.inTransaction()) throw new Error('Mongo read guards require an active transaction') } diff --git a/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts b/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts index bff77393e..a7810dd8b 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts @@ -45,25 +45,10 @@ export const MongoGridFsBucketName = 'overlayPayloads' const schemaVersion = 1 const maxUint64 = '18446744073709551615' const maxUint32 = 4294967295 -const paddedUint64Pattern = '^[0-9]{20}$' +const paddedUint64Pattern = String.raw`^\d{20}$` const hashPattern = '^[0-9a-f]{64}$' const uuidPattern = '^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$' -const safeStringPattern = '^[^\\x00]+$' - -function isWellFormed(value: string): boolean { - for (let index = 0; index < value.length; index += 1) { - const unit = value.charCodeAt(index) - if (unit >= 0xd800 && unit <= 0xdbff) { - if (index + 1 >= value.length) return false - const next = value.charCodeAt(index + 1) - if (next < 0xdc00 || next > 0xdfff) return false - index += 1 - } else if (unit >= 0xdc00 && unit <= 0xdfff) { - return false - } - } - return true -} +const safeStringPattern = String.raw`^[^\x00]+$` function validPart(value: string): string { if ( @@ -71,7 +56,7 @@ function validPart(value: string): string { value.length === 0 || Buffer.byteLength(value, 'utf8') > 1024 || value.includes('\u0000') || - !isWellFormed(value) + !value.isWellFormed() ) { throw new Error('Invalid Mongo record key component') } @@ -793,7 +778,7 @@ function stable(value: unknown): string { if (value !== null && typeof value === 'object') { const record = value as Record return `{${Object.keys(record) - .sort() + .sort((left, right) => left.localeCompare(right, 'en')) .map(key => `${JSON.stringify(key)}:${stable(record[key])}`) .join(',')}}` } @@ -844,27 +829,47 @@ export async function bootstrapMongoOverlay( } async function ensureCollection(db: Db, definition: MongoCollectionDefinition): Promise { + await createCollectionIfMissing(db, definition) + const actual = await db.listCollections({ name: definition.name }).next() + if (!collectionMatchesDefinition(actual, definition)) + throw new Error(`Incompatible Mongo Overlay validator for ${definition.name}`) + await ensureCollectionIndexes(db, definition) +} + +async function createCollectionIfMissing( + db: Db, + definition: MongoCollectionDefinition +): Promise { const existing = await db.listCollections({ name: definition.name }).next() - if (existing === null) { - try { - await db.createCollection(definition.name, { - validator: definition.validator, - validationLevel: 'strict', - validationAction: 'error', - collation: { locale: 'simple' } - }) - } catch (error) { - if ((error as { code?: number }).code !== 48) throw error - } + if (existing !== null) return + try { + await db.createCollection(definition.name, { + validator: definition.validator, + validationLevel: 'strict', + validationAction: 'error', + collation: { locale: 'simple' } + }) + } catch (error) { + if ((error as { code?: number }).code !== 48) throw error } - const actual = await db.listCollections({ name: definition.name }).next() - if ( - actual === null || - stable(actual.options?.validator) !== stable(definition.validator) || - (actual.options?.collation !== undefined && - stable(actual.options.collation) !== stable({ locale: 'simple' })) +} + +function collectionMatchesDefinition( + actual: CollectionInfo | null, + definition: MongoCollectionDefinition +): boolean { + return ( + actual !== null && + stable(actual.options?.validator) === stable(definition.validator) && + (actual.options?.collation === undefined || + stable(actual.options.collation) === stable({ locale: 'simple' })) ) - throw new Error(`Incompatible Mongo Overlay validator for ${definition.name}`) +} + +async function ensureCollectionIndexes( + db: Db, + definition: MongoCollectionDefinition +): Promise { const collection = db.collection(definition.name) for (const expected of definition.indexes) { const { key, ...options } = expected @@ -876,19 +881,30 @@ async function ensureCollection(db: Db, definition: MongoCollectionDefinition): } const actualIndexes = await collection.listIndexes().toArray() for (const expected of definition.indexes) { - const actualIndex = actualIndexes.find(candidate => candidate.name === expected.name) if ( - actualIndex === undefined || - stable(actualIndex.key) !== stable(expected.key) || - Boolean(actualIndex.unique) !== Boolean(expected.unique) || - stable(actualIndex.partialFilterExpression) !== stable(expected.partialFilterExpression) || - Boolean(actualIndex.sparse) !== Boolean(expected.sparse) || - actualIndex.expireAfterSeconds !== expected.expireAfterSeconds + !indexMatchesDefinition( + actualIndexes.find(candidate => candidate.name === expected.name), + expected + ) ) throw new Error(`Incompatible Mongo Overlay index for ${definition.name}:${expected.name}`) } } +function indexMatchesDefinition( + actualIndex: Document | undefined, + expected: IndexDescription +): boolean { + return ( + actualIndex !== undefined && + stable(actualIndex.key) === stable(expected.key) && + Boolean(actualIndex.unique) === Boolean(expected.unique) && + stable(actualIndex.partialFilterExpression) === stable(expected.partialFilterExpression) && + Boolean(actualIndex.sparse) === Boolean(expected.sparse) && + actualIndex.expireAfterSeconds === expected.expireAfterSeconds + ) +} + async function ensureGridFs(db: Db): Promise { const filesName = `${MongoGridFsBucketName}.files` const chunksName = `${MongoGridFsBucketName}.chunks` diff --git a/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts b/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts index 8788aac15..7464ac30c 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts @@ -122,6 +122,18 @@ class Budget { } } +function cloneRequest(request: MongoTransactionRequest): MongoTransactionRequest { + return { + key: { ...request.key, scope: { ...request.key.scope } }, + identity: { + ...request.identity, + scope: { ...request.identity.scope }, + topics: request.identity.topics.map(topic => ({ ...topic })) + }, + receipt: copyReceipt(request.receipt) + } +} + function copyReceipt(receipt: AdmissionReceipt): AdmissionReceipt { if ( receipt.durability !== 'atomic-local' || @@ -233,11 +245,10 @@ export class MongoTransactionRunner { await attempt.session.commitTransaction({ timeoutMS: budget.remaining() }) await this.release(attempt) return { state: 'committed', receipt: copyReceipt(attempt.receipt) } - } catch (error) { + } catch { // Unknown wins over any accompanying transient label. Never rerun its body. // Even an unexpected unlabeled failure after invocation stays pending. // Reconciliation, not error wording, determines its final outcome. - void error attempt.phase = 'unknown' if (budget.signal.aborted) break } @@ -266,66 +277,17 @@ export class MongoTransactionRunner { } async run(request: MongoTransactionRequest, body: (context: MongoTransactionContext) => Promise, options: MongoTransactionOptions = {}): Promise { - request = { - key: { ...request.key, scope: { ...request.key.scope } }, - identity: { ...request.identity, scope: { ...request.identity.scope }, topics: request.identity.topics.map(topic => ({ ...topic })) }, - receipt: copyReceipt(request.receipt) - } + request = cloneRequest(request) const id = this.id(request.key) - if (admissionSemanticDigest(request.identity) !== request.key.semanticDigest || mongoNodeKey(request.identity.scope) !== mongoNodeKey(this.scope)) - return { state: 'rejected', code: 'digest-mismatch' } const receipt = copyReceipt(request.receipt) - if (receipt.operationId !== request.key.operationId || receipt.semanticDigest !== request.key.semanticDigest) - return { state: 'rejected', code: 'invalid-plan' } + const rejected = this.rejectRun(request, receipt) + if (rejected !== undefined) return rejected const budget = new Budget(options) this.calls += 1 try { for (let bodyIndex = 0; bodyIndex < this.maxBodyAttempts; bodyIndex += 1) { - let previous = await this.read(id, budget) - if (previous === null) { - try { - await this.collection().insertOne({ _id: id, schemaVersion: 1, ...this.scope, operationId: request.key.operationId, semanticDigest: request.key.semanticDigest, txid: request.identity.txid, state: 'aborted', attemptId: randomUUID(), leaseOwner: this.owner, leaseToken: encodeMongoUint64('0'), leaseUntil: new Date(0), guard: randomUUID(), createdAt: new Date(), updatedAt: new Date() }, { ...budget.options(), writeConcern: majority }) - } catch (error) { - if (!duplicateKey(error)) throw error - } - previous = await this.read(id, budget) - if (previous === null) throw new Error('Mongo operation claim was not visible') - } - const existing = this.result(previous, request.key) - if (existing.state !== 'aborted') return existing - const attempt = await this.claim(previous, request, receipt, budget) - if (attempt === null) { - const winner = await this.read(id, budget) - if (winner === null) throw new Error('Mongo operation claim disappeared') - const result = this.result(winner, request.key) - if (result.state !== 'aborted') return result - continue - } - const { session, operation } = attempt - let bodyActive = true - const context: MongoTransactionContext = { session, signal: budget.signal, options: () => { - if (!bodyActive || !session.inTransaction()) throw new Error('Mongo transaction body is no longer active') - return { session, ...budget.options() } - } } - try { - session.startTransaction({ readConcern: { level: 'snapshot' }, writeConcern: majority, readPreference: 'primary', maxCommitTimeMS: budget.remaining() }) - const guarded = await this.collection().updateOne({ ...this.fence(operation), $expr: { $gt: ['$leaseUntil', '$$NOW'] } }, { $set: { guard: randomUUID() } }, context.options()) - if (guarded.modifiedCount !== 1) throw new Error('Mongo transaction ownership lost') - // Never end a session while its body is still running. Trusted bodies - // await every bounded database operation and observe the context gate. - await body(context) - const saved = await this.collection().updateOne(this.fence(operation), { $set: { state: 'committed', receipt: new Binary(Buffer.from(JSON.stringify(receipt), 'utf8')), guard: randomUUID() }, $currentDate: { updatedAt: true } }, context.options()) - if (saved.modifiedCount !== 1) throw new Error('Mongo transaction ownership lost') - bodyActive = false - return await this.commit(attempt, budget) - } catch (error) { - bodyActive = false - await this.abort(attempt) - if (!hasLabel(error, 'TransientTransactionError') || bodyIndex + 1 >= this.maxBodyAttempts) throw error - } finally { - bodyActive = false - attempt.busy = false - } + const completed = await this.runBodyAttempt(id, request, receipt, body, budget, bodyIndex) + if (completed !== undefined) return completed } throw new Error('Mongo transaction attempt limit reached') } finally { @@ -334,6 +296,100 @@ export class MongoTransactionRunner { } } + private rejectRun( + request: MongoTransactionRequest, + receipt: AdmissionReceipt + ): AdmissionCommitResult | undefined { + if ( + admissionSemanticDigest(request.identity) !== request.key.semanticDigest || + mongoNodeKey(request.identity.scope) !== mongoNodeKey(this.scope) + ) + return { state: 'rejected', code: 'digest-mismatch' } + if (receipt.operationId !== request.key.operationId || receipt.semanticDigest !== request.key.semanticDigest) + return { state: 'rejected', code: 'invalid-plan' } + return undefined + } + + private async ensureClaimableRow( + id: string, + request: MongoTransactionRequest, + budget: Budget + ): Promise { + let previous = await this.read(id, budget) + if (previous === null) { + try { + await this.collection().insertOne({ _id: id, schemaVersion: 1, ...this.scope, operationId: request.key.operationId, semanticDigest: request.key.semanticDigest, txid: request.identity.txid, state: 'aborted', attemptId: randomUUID(), leaseOwner: this.owner, leaseToken: encodeMongoUint64('0'), leaseUntil: new Date(0), guard: randomUUID(), createdAt: new Date(), updatedAt: new Date() }, { ...budget.options(), writeConcern: majority }) + } catch (error) { + if (!duplicateKey(error)) throw error + } + previous = await this.read(id, budget) + if (previous === null) throw new Error('Mongo operation claim was not visible') + } + return previous + } + + private async observeClaimWinner( + id: string, + request: MongoTransactionRequest, + budget: Budget + ): Promise { + const winner = await this.read(id, budget) + if (winner === null) throw new Error('Mongo operation claim disappeared') + const result = this.result(winner, request.key) + return result.state === 'aborted' ? undefined : result + } + + private async executeTrustedBody( + attempt: Attempt, + receipt: AdmissionReceipt, + body: (context: MongoTransactionContext) => Promise, + budget: Budget, + bodyIndex: number + ): Promise { + const { session, operation } = attempt + let bodyActive = true + const context: MongoTransactionContext = { session, signal: budget.signal, options: () => { + if (!bodyActive || !session.inTransaction()) throw new Error('Mongo transaction body is no longer active') + return { session, ...budget.options() } + } } + try { + session.startTransaction({ readConcern: { level: 'snapshot' }, writeConcern: majority, readPreference: 'primary', maxCommitTimeMS: budget.remaining() }) + const guarded = await this.collection().updateOne({ ...this.fence(operation), $expr: { $gt: ['$leaseUntil', '$$NOW'] } }, { $set: { guard: randomUUID() } }, context.options()) + if (guarded.modifiedCount !== 1) throw new Error('Mongo transaction ownership lost') + // Never end a session while its body is still running. Trusted bodies + // await every bounded database operation and observe the context gate. + await body(context) + const saved = await this.collection().updateOne(this.fence(operation), { $set: { state: 'committed', receipt: new Binary(Buffer.from(JSON.stringify(receipt), 'utf8')), guard: randomUUID() }, $currentDate: { updatedAt: true } }, context.options()) + if (saved.modifiedCount !== 1) throw new Error('Mongo transaction ownership lost') + bodyActive = false + return await this.commit(attempt, budget) + } catch (error) { + bodyActive = false + await this.abort(attempt) + if (!hasLabel(error, 'TransientTransactionError') || bodyIndex + 1 >= this.maxBodyAttempts) throw error + return undefined + } finally { + bodyActive = false + attempt.busy = false + } + } + + private async runBodyAttempt( + id: string, + request: MongoTransactionRequest, + receipt: AdmissionReceipt, + body: (context: MongoTransactionContext) => Promise, + budget: Budget, + bodyIndex: number + ): Promise { + const previous = await this.ensureClaimableRow(id, request, budget) + const existing = this.result(previous, request.key) + if (existing.state !== 'aborted') return existing + const attempt = await this.claim(previous, request, receipt, budget) + if (attempt === null) return await this.observeClaimWinner(id, request, budget) + return await this.executeTrustedBody(attempt, receipt, body, budget, bodyIndex) + } + async reconcile(key: AdmissionOperationKey, attemptId?: string, options: MongoTransactionOptions = {}): Promise { key = { ...key, scope: { ...key.scope } } const id = this.id(key) From 87cf2ed7d30cd860c55d5cbdcdb1b23adde4fbb2 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:21:52 -0500 Subject: [PATCH 021/168] fix(sonar): throw lookup abort instead of Promise.reject S7746 on LookupResolver: prefer throw over return Promise.reject in the facilitator start chain. --- packages/sdk/src/overlay-tools/LookupResolver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sdk/src/overlay-tools/LookupResolver.ts b/packages/sdk/src/overlay-tools/LookupResolver.ts index 047372ba1..d6789beaf 100644 --- a/packages/sdk/src/overlay-tools/LookupResolver.ts +++ b/packages/sdk/src/overlay-tools/LookupResolver.ts @@ -1825,7 +1825,7 @@ export default class LookupResolver { // Start the custom facilitator in a promise chain so synchronous throws // become rejections governed by the same wall-clock deadline. const lookupPromise = Promise.resolve().then(() => { - if (signal?.aborted === true) return Promise.reject(lookupAbortError()) + if (signal?.aborted === true) throw lookupAbortError() return this.facilitator.lookup(host, question, timeout, controller.signal, requestOptions) }) lookupPromise.catch(() => { From 8ab83644029bfc829577f5993cbe6fa1a830d52d Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:26:13 -0500 Subject: [PATCH 022/168] fix(wallet-toolbox): skip overlay identity when services are absent discoverOverlayCertificates called getServices(), which throws on wallets constructed without services and broke BRC-100 discoverBy* conformance. Missing chain context now yields no identities instead of throwing. --- packages/wallet/wallet-toolbox/src/Wallet.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/wallet/wallet-toolbox/src/Wallet.ts b/packages/wallet/wallet-toolbox/src/Wallet.ts index e2d84c06d..e1685ecb6 100644 --- a/packages/wallet/wallet-toolbox/src/Wallet.ts +++ b/packages/wallet/wallet-toolbox/src/Wallet.ts @@ -835,7 +835,10 @@ export class Wallet implements WalletInterface, ProtoWallet { now: number ): Promise { // Use the wallet's existing network/chain configuration, never the overlay host's verdict. - const chainTracker = await this.getServices().getChainTracker() + // Wallets constructed without services have no chain tracker; do not throw and do not + // emit identities that cannot be verified. + if (this.services == null) return [] + const chainTracker = await this.services.getChainTracker() if (this._identityEvidenceClosed) return [] const chainNamespace = `wallet:${this.chain}` if ( From 9b77db89f30daba5a872585ff5e9947f88df0100 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:26:13 -0500 Subject: [PATCH 023/168] fix(ci): override js-yaml 3.15.2 for GHSA-2883 Jest's istanbul graph still resolved 3.15.1, which fails pnpm audit --audit-level=high. --- governance/dependency-release-policy.json | 6 ++++++ governance/repository-health/exceptions.json | 14 ++++++++++++++ pnpm-lock.yaml | 11 ++++++----- pnpm-workspace.yaml | 2 ++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/governance/dependency-release-policy.json b/governance/dependency-release-policy.json index 0f41d8a89..39fc7055f 100644 --- a/governance/dependency-release-policy.json +++ b/governance/dependency-release-policy.json @@ -163,6 +163,12 @@ "value": "3.3.18", "exceptionId": "nanoid-postcss-advisory-override" }, + { + "source": "pnpm-workspace.yaml", + "selector": "js-yaml@<3.15.2", + "value": "3.15.2", + "exceptionId": "js-yaml-jest-advisory-override" + }, { "source": "pnpm-workspace.yaml", "selector": "toml@<4.2.0", diff --git a/governance/repository-health/exceptions.json b/governance/repository-health/exceptions.json index b1d5c70c5..f5b4d306d 100644 --- a/governance/repository-health/exceptions.json +++ b/governance/repository-health/exceptions.json @@ -157,6 +157,20 @@ "reviewBy": "2026-09-13", "removeWhen": "Remove when the supported Vite and PostCSS dependency graph naturally resolves nanoid 3.3.18 or newer and the frozen audit remains green." }, + { + "id": "js-yaml-jest-advisory-override", + "category": "override", + "target": "pnpm-workspace.yaml override js-yaml releases below 3.15.2", + "owner": "ts-stack-maintainers", + "reason": "Jest's istanbul/nyc coverage graph still resolves js-yaml 3.15.1. GHSA-2883-xcg3-v3hh is fixed in 3.15.2 with no public API change on the 3.x line. Selecting 3.15.2 is the narrowest compatible remediation and leaves runtime application packages on js-yaml 4.x overrides unchanged.", + "evidence": [ + "pnpm-workspace.yaml#overrides", + "https://github.com/advisories/GHSA-2883-xcg3-v3hh" + ], + "created": "2026-09-09", + "reviewBy": "2026-10-09", + "removeWhen": "Remove when the supported Jest coverage graph naturally resolves js-yaml 3.15.2 or newer and the frozen audit remains green." + }, { "id": "toml-frontmatter-advisory-override", "category": "override", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32cc1896c..b9ea4d64b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,6 +7,7 @@ settings: overrides: brace-expansion@<5.0.9: 5.0.9 nanoid@<3.3.18: 3.3.18 + js-yaml@<3.15.2: 3.15.2 toml@<4.2.0: 4.2.0 typed-rest-client@2.3.1>qs: 6.15.3 @@ -6421,8 +6422,8 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - js-yaml@3.15.1: - resolution: {integrity: sha512-S99WuO3HlhO3XN41EtYUNl9zzXjoJx7QvmipxsJVxtCBT0YHEFy+iOJhjSvrmV12nYhWpZaM8lPHkJm0yUMbag==} + js-yaml@3.15.2: + resolution: {integrity: sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==} hasBin: true jsc-safe-url@0.2.4: @@ -9679,7 +9680,7 @@ snapshots: camelcase: 5.3.1 find-up: 4.1.0 get-package-type: 0.1.0 - js-yaml: 3.15.1 + js-yaml: 3.15.2 resolve-from: 5.0.0 '@istanbuljs/schema@0.1.6': {} @@ -12958,7 +12959,7 @@ snapshots: gray-matter@4.0.3: dependencies: - js-yaml: 3.15.1 + js-yaml: 3.15.2 kind-of: 6.0.3 section-matter: 1.0.0 strip-bom-string: 1.0.0 @@ -13742,7 +13743,7 @@ snapshots: js-tokens@4.0.0: {} - js-yaml@3.15.1: + js-yaml@3.15.2: dependencies: argparse: 1.0.10 esprima: 4.0.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b0a4e518c..8dc8eea40 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -34,6 +34,8 @@ overrides: # Vite's PostCSS graph permits nanoid 3 releases older than the security # fix. Select the first patched 3.x release without changing its public API. nanoid@<3.3.18: 3.3.18 + # Jest's istanbul/nyc graph still resolves js-yaml 3.15.1 (GHSA-2883-xcg3-v3hh). + js-yaml@<3.15.2: 3.15.2 # remark-mdx-frontmatter 5.2.0 still constrains TOML to vulnerable 3.x even # though it only consumes the compatible parse() API. webpack-cli already # accepts TOML 4; keep both paths on the first release fixing the advisories. From 1e2ec60741a254773cbcc5d7b7302e0742003014 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:26:16 -0500 Subject: [PATCH 024/168] fix(ci): override js-yaml 3.15.2 for GHSA-2883 Jest's istanbul graph still resolved 3.15.1, which fails pnpm audit --audit-level=high. --- governance/dependency-release-policy.json | 6 +++ governance/repository-health/exceptions.json | 14 +++++++ pnpm-lock.yaml | 39 +++++++++++++------- pnpm-workspace.yaml | 2 + 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/governance/dependency-release-policy.json b/governance/dependency-release-policy.json index 0f41d8a89..39fc7055f 100644 --- a/governance/dependency-release-policy.json +++ b/governance/dependency-release-policy.json @@ -163,6 +163,12 @@ "value": "3.3.18", "exceptionId": "nanoid-postcss-advisory-override" }, + { + "source": "pnpm-workspace.yaml", + "selector": "js-yaml@<3.15.2", + "value": "3.15.2", + "exceptionId": "js-yaml-jest-advisory-override" + }, { "source": "pnpm-workspace.yaml", "selector": "toml@<4.2.0", diff --git a/governance/repository-health/exceptions.json b/governance/repository-health/exceptions.json index b1d5c70c5..f5b4d306d 100644 --- a/governance/repository-health/exceptions.json +++ b/governance/repository-health/exceptions.json @@ -157,6 +157,20 @@ "reviewBy": "2026-09-13", "removeWhen": "Remove when the supported Vite and PostCSS dependency graph naturally resolves nanoid 3.3.18 or newer and the frozen audit remains green." }, + { + "id": "js-yaml-jest-advisory-override", + "category": "override", + "target": "pnpm-workspace.yaml override js-yaml releases below 3.15.2", + "owner": "ts-stack-maintainers", + "reason": "Jest's istanbul/nyc coverage graph still resolves js-yaml 3.15.1. GHSA-2883-xcg3-v3hh is fixed in 3.15.2 with no public API change on the 3.x line. Selecting 3.15.2 is the narrowest compatible remediation and leaves runtime application packages on js-yaml 4.x overrides unchanged.", + "evidence": [ + "pnpm-workspace.yaml#overrides", + "https://github.com/advisories/GHSA-2883-xcg3-v3hh" + ], + "created": "2026-09-09", + "reviewBy": "2026-10-09", + "removeWhen": "Remove when the supported Jest coverage graph naturally resolves js-yaml 3.15.2 or newer and the frozen audit remains green." + }, { "id": "toml-frontmatter-advisory-override", "category": "override", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32cc1896c..585bacf5d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,6 +7,7 @@ settings: overrides: brace-expansion@<5.0.9: 5.0.9 nanoid@<3.3.18: 3.3.18 + js-yaml@<3.15.2: 3.15.2 toml@<4.2.0: 4.2.0 typed-rest-client@2.3.1>qs: 6.15.3 @@ -79,7 +80,7 @@ importers: version: 8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) vitest: specifier: ^4.1.10 - version: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + version: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(esbuild@0.28.1)(jsdom@26.1.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) conformance/runner: dependencies: @@ -522,7 +523,7 @@ importers: version: '@typescript/typescript6@6.0.2' vitest: specifier: ^4.1.10 - version: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + version: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(esbuild@0.28.1)(jsdom@26.1.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) packages/helpers/simple: dependencies: @@ -852,7 +853,7 @@ importers: version: '@typescript/typescript6@6.0.2' vitest: specifier: ^4.1.10 - version: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + version: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(esbuild@0.28.1)(jsdom@26.1.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) packages/middleware/auth: devDependencies: @@ -1916,7 +1917,7 @@ importers: version: 8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) vitest: specifier: ^4.1.10 - version: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + version: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(esbuild@0.28.1)(jsdom@26.1.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) packages/wallet/wallet-toolbox/mobile: dependencies: @@ -1971,7 +1972,7 @@ importers: version: '@typescript/typescript6@6.0.2' vitest: specifier: ^4.1.10 - version: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + version: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(esbuild@0.28.1)(jsdom@26.1.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) packages: @@ -6421,8 +6422,8 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - js-yaml@3.15.1: - resolution: {integrity: sha512-S99WuO3HlhO3XN41EtYUNl9zzXjoJx7QvmipxsJVxtCBT0YHEFy+iOJhjSvrmV12nYhWpZaM8lPHkJm0yUMbag==} + js-yaml@3.15.2: + resolution: {integrity: sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==} hasBin: true jsc-safe-url@0.2.4: @@ -8491,7 +8492,6 @@ packages: '@vitest/ui': 4.1.10 happy-dom: '*' jsdom: '*' - vite: ^6.0.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: '@edge-runtime/vm': optional: true @@ -9679,7 +9679,7 @@ snapshots: camelcase: 5.3.1 find-up: 4.1.0 get-package-type: 0.1.0 - js-yaml: 3.15.1 + js-yaml: 3.15.2 resolve-from: 5.0.0 '@istanbuljs/schema@0.1.6': {} @@ -10941,7 +10941,7 @@ snapshots: '@stryker-mutator/util': 9.6.1 semver: 7.8.5 tslib: 2.8.1 - vitest: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + vitest: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(esbuild@0.28.1)(jsdom@26.1.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) '@testing-library/dom@10.4.1': dependencies: @@ -11455,7 +11455,7 @@ snapshots: obug: 2.1.4 std-env: 4.2.0 tinyrainbow: 3.1.0 - vitest: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + vitest: 4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(esbuild@0.28.1)(jsdom@26.1.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) '@vitest/expect@4.1.10': dependencies: @@ -12958,7 +12958,7 @@ snapshots: gray-matter@4.0.3: dependencies: - js-yaml: 3.15.1 + js-yaml: 3.15.2 kind-of: 6.0.3 section-matter: 1.0.0 strip-bom-string: 1.0.0 @@ -13742,7 +13742,7 @@ snapshots: js-tokens@4.0.0: {} - js-yaml@3.15.1: + js-yaml@3.15.2: dependencies: argparse: 1.0.10 esprima: 4.0.1 @@ -16328,7 +16328,7 @@ snapshots: tsx: 4.23.1 yaml: 2.9.0 - vitest@4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(jsdom@26.1.0)(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)): + vitest@4.1.10(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(esbuild@0.28.1)(jsdom@26.1.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0): dependencies: '@vitest/expect': 4.1.10 '@vitest/mocker': 4.1.10(vite@8.1.5(@types/node@26.1.2)(esbuild@0.28.1)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) @@ -16355,7 +16355,18 @@ snapshots: '@vitest/coverage-v8': 4.1.10(vitest@4.1.10) jsdom: 26.1.0 transitivePeerDependencies: + - '@vitejs/devtools' + - esbuild + - jiti + - less - msw + - sass + - sass-embedded + - stylus + - sugarss + - terser + - tsx + - yaml vlq@1.0.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b0a4e518c..8dc8eea40 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -34,6 +34,8 @@ overrides: # Vite's PostCSS graph permits nanoid 3 releases older than the security # fix. Select the first patched 3.x release without changing its public API. nanoid@<3.3.18: 3.3.18 + # Jest's istanbul/nyc graph still resolves js-yaml 3.15.1 (GHSA-2883-xcg3-v3hh). + js-yaml@<3.15.2: 3.15.2 # remark-mdx-frontmatter 5.2.0 still constrains TOML to vulnerable 3.x even # though it only consumes the compatible parse() API. webpack-cli already # accepts TOML 4; keep both paths on the first release fixing the advisories. From 4daf07fed2c98ede05bb03f30225ca3d1989151f Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:26:18 -0500 Subject: [PATCH 025/168] fix(ci): align sdk 2.5.0 health baseline and stack facts Merge from main left baselines.json on 2.4.2 while package.json is 2.5.0. --- docs/reference/stack-facts.md | 2 +- governance/repository-health/baselines.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 56b67c1a9..713c4b77d 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -62,7 +62,7 @@ authorized release action. | overlays | `@bsv/overlay-discovery-services` | `2.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | | overlays | `@bsv/overlay-express` | `2.6.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | | overlays | `@bsv/overlay-topics` | `1.7.2` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | -| sdk | `@bsv/sdk` | `2.4.2` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | +| sdk | `@bsv/sdk` | `2.5.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | | sdk | `@bsv/verifast` | `0.3.5` | wasm-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global, wasm-worker | browser, node, umd, wasm, worker | `>=22` | [packages/verifast](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/verifast) | | wallet | `@bsv/btms` | `1.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | | wallet | `@bsv/btms-permission-module` | `1.1.4` | node-library | node-esm | node | `>=22` | [packages/wallet/btms-permission-module](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms-permission-module) | diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index 3e6ca49b6..ba31c6005 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -322,7 +322,7 @@ "@bsv/overlay-discovery-services": "2.2.2", "@bsv/overlay-express": "2.6.2", "@bsv/overlay-topics": "1.7.2", - "@bsv/sdk": "2.4.2", + "@bsv/sdk": "2.5.0", "@bsv/verifast": "0.3.5", "@bsv/btms": "1.2.2", "@bsv/btms-permission-module": "1.1.4", From 16c0ec45cadaead42a9328c6fc3de677b989ec28 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:26:19 -0500 Subject: [PATCH 026/168] fix(ci): override js-yaml 3.15.2 for GHSA-2883 Jest's istanbul graph still resolved 3.15.1, which fails pnpm audit --audit-level=high. --- governance/dependency-release-policy.json | 6 ++++++ governance/repository-health/exceptions.json | 14 ++++++++++++++ pnpm-lock.yaml | 11 ++++++----- pnpm-workspace.yaml | 2 ++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/governance/dependency-release-policy.json b/governance/dependency-release-policy.json index 0f41d8a89..39fc7055f 100644 --- a/governance/dependency-release-policy.json +++ b/governance/dependency-release-policy.json @@ -163,6 +163,12 @@ "value": "3.3.18", "exceptionId": "nanoid-postcss-advisory-override" }, + { + "source": "pnpm-workspace.yaml", + "selector": "js-yaml@<3.15.2", + "value": "3.15.2", + "exceptionId": "js-yaml-jest-advisory-override" + }, { "source": "pnpm-workspace.yaml", "selector": "toml@<4.2.0", diff --git a/governance/repository-health/exceptions.json b/governance/repository-health/exceptions.json index b1d5c70c5..f5b4d306d 100644 --- a/governance/repository-health/exceptions.json +++ b/governance/repository-health/exceptions.json @@ -157,6 +157,20 @@ "reviewBy": "2026-09-13", "removeWhen": "Remove when the supported Vite and PostCSS dependency graph naturally resolves nanoid 3.3.18 or newer and the frozen audit remains green." }, + { + "id": "js-yaml-jest-advisory-override", + "category": "override", + "target": "pnpm-workspace.yaml override js-yaml releases below 3.15.2", + "owner": "ts-stack-maintainers", + "reason": "Jest's istanbul/nyc coverage graph still resolves js-yaml 3.15.1. GHSA-2883-xcg3-v3hh is fixed in 3.15.2 with no public API change on the 3.x line. Selecting 3.15.2 is the narrowest compatible remediation and leaves runtime application packages on js-yaml 4.x overrides unchanged.", + "evidence": [ + "pnpm-workspace.yaml#overrides", + "https://github.com/advisories/GHSA-2883-xcg3-v3hh" + ], + "created": "2026-09-09", + "reviewBy": "2026-10-09", + "removeWhen": "Remove when the supported Jest coverage graph naturally resolves js-yaml 3.15.2 or newer and the frozen audit remains green." + }, { "id": "toml-frontmatter-advisory-override", "category": "override", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index caba6275a..3368f9c42 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,6 +7,7 @@ settings: overrides: brace-expansion@<5.0.9: 5.0.9 nanoid@<3.3.18: 3.3.18 + js-yaml@<3.15.2: 3.15.2 toml@<4.2.0: 4.2.0 typed-rest-client@2.3.1>qs: 6.15.3 @@ -6427,8 +6428,8 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - js-yaml@3.15.1: - resolution: {integrity: sha512-S99WuO3HlhO3XN41EtYUNl9zzXjoJx7QvmipxsJVxtCBT0YHEFy+iOJhjSvrmV12nYhWpZaM8lPHkJm0yUMbag==} + js-yaml@3.15.2: + resolution: {integrity: sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==} hasBin: true jsc-safe-url@0.2.4: @@ -9685,7 +9686,7 @@ snapshots: camelcase: 5.3.1 find-up: 4.1.0 get-package-type: 0.1.0 - js-yaml: 3.15.1 + js-yaml: 3.15.2 resolve-from: 5.0.0 '@istanbuljs/schema@0.1.6': {} @@ -12964,7 +12965,7 @@ snapshots: gray-matter@4.0.3: dependencies: - js-yaml: 3.15.1 + js-yaml: 3.15.2 kind-of: 6.0.3 section-matter: 1.0.0 strip-bom-string: 1.0.0 @@ -13748,7 +13749,7 @@ snapshots: js-tokens@4.0.0: {} - js-yaml@3.15.1: + js-yaml@3.15.2: dependencies: argparse: 1.0.10 esprima: 4.0.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b0a4e518c..8dc8eea40 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -34,6 +34,8 @@ overrides: # Vite's PostCSS graph permits nanoid 3 releases older than the security # fix. Select the first patched 3.x release without changing its public API. nanoid@<3.3.18: 3.3.18 + # Jest's istanbul/nyc graph still resolves js-yaml 3.15.1 (GHSA-2883-xcg3-v3hh). + js-yaml@<3.15.2: 3.15.2 # remark-mdx-frontmatter 5.2.0 still constrains TOML to vulnerable 3.x even # though it only consumes the compatible parse() API. webpack-cli already # accepts TOML 4; keep both paths on the first release fixing the advisories. From ea1e532946070dba2421ccc4836c28d3cdf50471 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:26:21 -0500 Subject: [PATCH 027/168] fix(ci): override js-yaml 3.15.2 for GHSA-2883 Jest's istanbul graph still resolved 3.15.1, which fails pnpm audit --audit-level=high. --- governance/dependency-release-policy.json | 6 ++++++ governance/repository-health/exceptions.json | 14 ++++++++++++++ pnpm-lock.yaml | 11 ++++++----- pnpm-workspace.yaml | 2 ++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/governance/dependency-release-policy.json b/governance/dependency-release-policy.json index 0f41d8a89..39fc7055f 100644 --- a/governance/dependency-release-policy.json +++ b/governance/dependency-release-policy.json @@ -163,6 +163,12 @@ "value": "3.3.18", "exceptionId": "nanoid-postcss-advisory-override" }, + { + "source": "pnpm-workspace.yaml", + "selector": "js-yaml@<3.15.2", + "value": "3.15.2", + "exceptionId": "js-yaml-jest-advisory-override" + }, { "source": "pnpm-workspace.yaml", "selector": "toml@<4.2.0", diff --git a/governance/repository-health/exceptions.json b/governance/repository-health/exceptions.json index b1d5c70c5..f5b4d306d 100644 --- a/governance/repository-health/exceptions.json +++ b/governance/repository-health/exceptions.json @@ -157,6 +157,20 @@ "reviewBy": "2026-09-13", "removeWhen": "Remove when the supported Vite and PostCSS dependency graph naturally resolves nanoid 3.3.18 or newer and the frozen audit remains green." }, + { + "id": "js-yaml-jest-advisory-override", + "category": "override", + "target": "pnpm-workspace.yaml override js-yaml releases below 3.15.2", + "owner": "ts-stack-maintainers", + "reason": "Jest's istanbul/nyc coverage graph still resolves js-yaml 3.15.1. GHSA-2883-xcg3-v3hh is fixed in 3.15.2 with no public API change on the 3.x line. Selecting 3.15.2 is the narrowest compatible remediation and leaves runtime application packages on js-yaml 4.x overrides unchanged.", + "evidence": [ + "pnpm-workspace.yaml#overrides", + "https://github.com/advisories/GHSA-2883-xcg3-v3hh" + ], + "created": "2026-09-09", + "reviewBy": "2026-10-09", + "removeWhen": "Remove when the supported Jest coverage graph naturally resolves js-yaml 3.15.2 or newer and the frozen audit remains green." + }, { "id": "toml-frontmatter-advisory-override", "category": "override", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32cc1896c..b9ea4d64b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,6 +7,7 @@ settings: overrides: brace-expansion@<5.0.9: 5.0.9 nanoid@<3.3.18: 3.3.18 + js-yaml@<3.15.2: 3.15.2 toml@<4.2.0: 4.2.0 typed-rest-client@2.3.1>qs: 6.15.3 @@ -6421,8 +6422,8 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - js-yaml@3.15.1: - resolution: {integrity: sha512-S99WuO3HlhO3XN41EtYUNl9zzXjoJx7QvmipxsJVxtCBT0YHEFy+iOJhjSvrmV12nYhWpZaM8lPHkJm0yUMbag==} + js-yaml@3.15.2: + resolution: {integrity: sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==} hasBin: true jsc-safe-url@0.2.4: @@ -9679,7 +9680,7 @@ snapshots: camelcase: 5.3.1 find-up: 4.1.0 get-package-type: 0.1.0 - js-yaml: 3.15.1 + js-yaml: 3.15.2 resolve-from: 5.0.0 '@istanbuljs/schema@0.1.6': {} @@ -12958,7 +12959,7 @@ snapshots: gray-matter@4.0.3: dependencies: - js-yaml: 3.15.1 + js-yaml: 3.15.2 kind-of: 6.0.3 section-matter: 1.0.0 strip-bom-string: 1.0.0 @@ -13742,7 +13743,7 @@ snapshots: js-tokens@4.0.0: {} - js-yaml@3.15.1: + js-yaml@3.15.2: dependencies: argparse: 1.0.10 esprima: 4.0.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b0a4e518c..8dc8eea40 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -34,6 +34,8 @@ overrides: # Vite's PostCSS graph permits nanoid 3 releases older than the security # fix. Select the first patched 3.x release without changing its public API. nanoid@<3.3.18: 3.3.18 + # Jest's istanbul/nyc graph still resolves js-yaml 3.15.1 (GHSA-2883-xcg3-v3hh). + js-yaml@<3.15.2: 3.15.2 # remark-mdx-frontmatter 5.2.0 still constrains TOML to vulnerable 3.x even # though it only consumes the compatible parse() API. webpack-cli already # accepts TOML 4; keep both paths on the first release fixing the advisories. From 4f6a88ccca6b5c6e10fee42d49243bf2e1ce6973 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:30:00 -0500 Subject: [PATCH 028/168] fix(ci): record js-yaml override removal rehearsal Health requires overrideRemovalReview.retainedCount to match the live registry. --- governance/dependency-release-policy.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/governance/dependency-release-policy.json b/governance/dependency-release-policy.json index 39fc7055f..0a78775cd 100644 --- a/governance/dependency-release-policy.json +++ b/governance/dependency-release-policy.json @@ -145,10 +145,10 @@ }, "overrideRemovalReview": { "reviewedAt": "2026-09-04", - "method": "Carried forward the complete 2026-08-30 removal rehearsal within its monthly review window and immediately reviewed the new TOML advisories against the frozen graph and current package metadata. Verified that remark-mdx-frontmatter 5.2.0 still requires TOML 3.x, consumes only parse(), webpack-cli supports TOML 4, and TOML 4.2.0 passes docs validation, tests, and the complete production build.", - "retainedCount": 21, - "result": "All 21 retained overrides still prevent a reproduced vulnerable transitive version or preserve an isolated reproducible toolchain closure. The new TOML 4.2.0 substitution removes both high-severity parser advisories; the frozen high-severity audit now reports only the two independently patched and governed image-size advisories.", - "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, or image-size dependency changes." + "method": "Carried forward the complete 2026-08-30 removal rehearsal within its monthly review window, reviewed the TOML advisories against the frozen graph, and immediately rehearsed adding the js-yaml 3.15.2 substitution for GHSA-2883-xcg3-v3hh. Regenerating the lock without that override still selects js-yaml 3.15.1 via Jest's istanbul/nyc graph; 3.15.2 is the first patched 3.x release and leaves the existing js-yaml 4.x Redocly override unchanged.", + "retainedCount": 22, + "result": "All 22 retained overrides still prevent a reproduced vulnerable transitive version or preserve an isolated reproducible toolchain closure. The js-yaml 3.15.2 substitution removes GHSA-2883-xcg3-v3hh from the frozen high-severity audit; image-size advisories remain independently patched.", + "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch/js-yaml, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, or image-size dependency changes." }, "overrideRegistry": [ { From b78ec98bc156383da4ce5f370316930a4a92f7fd Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:30:02 -0500 Subject: [PATCH 029/168] fix(ci): record js-yaml override removal rehearsal Health requires overrideRemovalReview.retainedCount to match the live registry. --- governance/dependency-release-policy.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/governance/dependency-release-policy.json b/governance/dependency-release-policy.json index 39fc7055f..0a78775cd 100644 --- a/governance/dependency-release-policy.json +++ b/governance/dependency-release-policy.json @@ -145,10 +145,10 @@ }, "overrideRemovalReview": { "reviewedAt": "2026-09-04", - "method": "Carried forward the complete 2026-08-30 removal rehearsal within its monthly review window and immediately reviewed the new TOML advisories against the frozen graph and current package metadata. Verified that remark-mdx-frontmatter 5.2.0 still requires TOML 3.x, consumes only parse(), webpack-cli supports TOML 4, and TOML 4.2.0 passes docs validation, tests, and the complete production build.", - "retainedCount": 21, - "result": "All 21 retained overrides still prevent a reproduced vulnerable transitive version or preserve an isolated reproducible toolchain closure. The new TOML 4.2.0 substitution removes both high-severity parser advisories; the frozen high-severity audit now reports only the two independently patched and governed image-size advisories.", - "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, or image-size dependency changes." + "method": "Carried forward the complete 2026-08-30 removal rehearsal within its monthly review window, reviewed the TOML advisories against the frozen graph, and immediately rehearsed adding the js-yaml 3.15.2 substitution for GHSA-2883-xcg3-v3hh. Regenerating the lock without that override still selects js-yaml 3.15.1 via Jest's istanbul/nyc graph; 3.15.2 is the first patched 3.x release and leaves the existing js-yaml 4.x Redocly override unchanged.", + "retainedCount": 22, + "result": "All 22 retained overrides still prevent a reproduced vulnerable transitive version or preserve an isolated reproducible toolchain closure. The js-yaml 3.15.2 substitution removes GHSA-2883-xcg3-v3hh from the frozen high-severity audit; image-size advisories remain independently patched.", + "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch/js-yaml, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, or image-size dependency changes." }, "overrideRegistry": [ { From 9fe97d2b6eb0633f8905c4f5cf175aa3dc141c20 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:30:04 -0500 Subject: [PATCH 030/168] fix(ci): record js-yaml override removal rehearsal Health requires overrideRemovalReview.retainedCount to match the live registry. --- governance/dependency-release-policy.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/governance/dependency-release-policy.json b/governance/dependency-release-policy.json index 39fc7055f..0a78775cd 100644 --- a/governance/dependency-release-policy.json +++ b/governance/dependency-release-policy.json @@ -145,10 +145,10 @@ }, "overrideRemovalReview": { "reviewedAt": "2026-09-04", - "method": "Carried forward the complete 2026-08-30 removal rehearsal within its monthly review window and immediately reviewed the new TOML advisories against the frozen graph and current package metadata. Verified that remark-mdx-frontmatter 5.2.0 still requires TOML 3.x, consumes only parse(), webpack-cli supports TOML 4, and TOML 4.2.0 passes docs validation, tests, and the complete production build.", - "retainedCount": 21, - "result": "All 21 retained overrides still prevent a reproduced vulnerable transitive version or preserve an isolated reproducible toolchain closure. The new TOML 4.2.0 substitution removes both high-severity parser advisories; the frozen high-severity audit now reports only the two independently patched and governed image-size advisories.", - "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, or image-size dependency changes." + "method": "Carried forward the complete 2026-08-30 removal rehearsal within its monthly review window, reviewed the TOML advisories against the frozen graph, and immediately rehearsed adding the js-yaml 3.15.2 substitution for GHSA-2883-xcg3-v3hh. Regenerating the lock without that override still selects js-yaml 3.15.1 via Jest's istanbul/nyc graph; 3.15.2 is the first patched 3.x release and leaves the existing js-yaml 4.x Redocly override unchanged.", + "retainedCount": 22, + "result": "All 22 retained overrides still prevent a reproduced vulnerable transitive version or preserve an isolated reproducible toolchain closure. The js-yaml 3.15.2 substitution removes GHSA-2883-xcg3-v3hh from the frozen high-severity audit; image-size advisories remain independently patched.", + "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch/js-yaml, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, or image-size dependency changes." }, "overrideRegistry": [ { From e5aeed505f14e7e1bd5ad4b0de61a7de01ecce3a Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:30:06 -0500 Subject: [PATCH 031/168] fix(ci): record js-yaml override removal rehearsal Health requires overrideRemovalReview.retainedCount to match the live registry. --- governance/dependency-release-policy.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/governance/dependency-release-policy.json b/governance/dependency-release-policy.json index 39fc7055f..0a78775cd 100644 --- a/governance/dependency-release-policy.json +++ b/governance/dependency-release-policy.json @@ -145,10 +145,10 @@ }, "overrideRemovalReview": { "reviewedAt": "2026-09-04", - "method": "Carried forward the complete 2026-08-30 removal rehearsal within its monthly review window and immediately reviewed the new TOML advisories against the frozen graph and current package metadata. Verified that remark-mdx-frontmatter 5.2.0 still requires TOML 3.x, consumes only parse(), webpack-cli supports TOML 4, and TOML 4.2.0 passes docs validation, tests, and the complete production build.", - "retainedCount": 21, - "result": "All 21 retained overrides still prevent a reproduced vulnerable transitive version or preserve an isolated reproducible toolchain closure. The new TOML 4.2.0 substitution removes both high-severity parser advisories; the frozen high-severity audit now reports only the two independently patched and governed image-size advisories.", - "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, or image-size dependency changes." + "method": "Carried forward the complete 2026-08-30 removal rehearsal within its monthly review window, reviewed the TOML advisories against the frozen graph, and immediately rehearsed adding the js-yaml 3.15.2 substitution for GHSA-2883-xcg3-v3hh. Regenerating the lock without that override still selects js-yaml 3.15.1 via Jest's istanbul/nyc graph; 3.15.2 is the first patched 3.x release and leaves the existing js-yaml 4.x Redocly override unchanged.", + "retainedCount": 22, + "result": "All 22 retained overrides still prevent a reproduced vulnerable transitive version or preserve an isolated reproducible toolchain closure. The js-yaml 3.15.2 substitution removes GHSA-2883-xcg3-v3hh from the frozen high-severity audit; image-size advisories remain independently patched.", + "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch/js-yaml, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, or image-size dependency changes." }, "overrideRegistry": [ { From 1a143ced4c52d6e3a5a4e3b549d060664a36ec6a Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:30:08 -0500 Subject: [PATCH 032/168] fix(ci): record js-yaml override removal rehearsal Health requires overrideRemovalReview.retainedCount to match the live registry. --- governance/dependency-release-policy.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/governance/dependency-release-policy.json b/governance/dependency-release-policy.json index 39fc7055f..0a78775cd 100644 --- a/governance/dependency-release-policy.json +++ b/governance/dependency-release-policy.json @@ -145,10 +145,10 @@ }, "overrideRemovalReview": { "reviewedAt": "2026-09-04", - "method": "Carried forward the complete 2026-08-30 removal rehearsal within its monthly review window and immediately reviewed the new TOML advisories against the frozen graph and current package metadata. Verified that remark-mdx-frontmatter 5.2.0 still requires TOML 3.x, consumes only parse(), webpack-cli supports TOML 4, and TOML 4.2.0 passes docs validation, tests, and the complete production build.", - "retainedCount": 21, - "result": "All 21 retained overrides still prevent a reproduced vulnerable transitive version or preserve an isolated reproducible toolchain closure. The new TOML 4.2.0 substitution removes both high-severity parser advisories; the frozen high-severity audit now reports only the two independently patched and governed image-size advisories.", - "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, or image-size dependency changes." + "method": "Carried forward the complete 2026-08-30 removal rehearsal within its monthly review window, reviewed the TOML advisories against the frozen graph, and immediately rehearsed adding the js-yaml 3.15.2 substitution for GHSA-2883-xcg3-v3hh. Regenerating the lock without that override still selects js-yaml 3.15.1 via Jest's istanbul/nyc graph; 3.15.2 is the first patched 3.x release and leaves the existing js-yaml 4.x Redocly override unchanged.", + "retainedCount": 22, + "result": "All 22 retained overrides still prevent a reproduced vulnerable transitive version or preserve an isolated reproducible toolchain closure. The js-yaml 3.15.2 substitution removes GHSA-2883-xcg3-v3hh from the frozen high-severity audit; image-size advisories remain independently patched.", + "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch/js-yaml, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, or image-size dependency changes." }, "overrideRegistry": [ { From 103ce20f8143d37680c5d1a05abcfd2f6348e337 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:36:34 -0500 Subject: [PATCH 033/168] fix(ci): ratchet override count for js-yaml 3.15.2 --- scripts/dependency-release-governance.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/dependency-release-governance.test.mjs b/scripts/dependency-release-governance.test.mjs index e8e13bb31..deb2d70d7 100644 --- a/scripts/dependency-release-governance.test.mjs +++ b/scripts/dependency-release-governance.test.mjs @@ -21,7 +21,8 @@ test('dependency and release governance is internally complete', () => { assert.deepEqual(validateDependencyReleaseGovernance(), []) const overrides = collectOverrides() - assert.equal(overrides.length, 21) + assert.equal(overrides.length, 22) + assert.equal(overrides.filter(entry => entry.selector === 'js-yaml@<3.15.2').length, 1) assert.equal(overrides.filter(entry => entry.selector === 'gaxios').length, 8) assert.equal(overrides.filter(entry => entry.selector === 'uuid').length, 3) assert.equal(overrides.filter(entry => entry.selector === 'brace-expansion').length, 4) From 60b5b260453232e677872d33f77155353783c940 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:36:37 -0500 Subject: [PATCH 034/168] fix(ci): ratchet override count for js-yaml 3.15.2 --- scripts/dependency-release-governance.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/dependency-release-governance.test.mjs b/scripts/dependency-release-governance.test.mjs index e8e13bb31..deb2d70d7 100644 --- a/scripts/dependency-release-governance.test.mjs +++ b/scripts/dependency-release-governance.test.mjs @@ -21,7 +21,8 @@ test('dependency and release governance is internally complete', () => { assert.deepEqual(validateDependencyReleaseGovernance(), []) const overrides = collectOverrides() - assert.equal(overrides.length, 21) + assert.equal(overrides.length, 22) + assert.equal(overrides.filter(entry => entry.selector === 'js-yaml@<3.15.2').length, 1) assert.equal(overrides.filter(entry => entry.selector === 'gaxios').length, 8) assert.equal(overrides.filter(entry => entry.selector === 'uuid').length, 3) assert.equal(overrides.filter(entry => entry.selector === 'brace-expansion').length, 4) From 75c8c5c2cd030958c4bce5d227643a13209dd8ae Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:36:39 -0500 Subject: [PATCH 035/168] fix(ci): ratchet override count for js-yaml 3.15.2 --- scripts/dependency-release-governance.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/dependency-release-governance.test.mjs b/scripts/dependency-release-governance.test.mjs index e8e13bb31..deb2d70d7 100644 --- a/scripts/dependency-release-governance.test.mjs +++ b/scripts/dependency-release-governance.test.mjs @@ -21,7 +21,8 @@ test('dependency and release governance is internally complete', () => { assert.deepEqual(validateDependencyReleaseGovernance(), []) const overrides = collectOverrides() - assert.equal(overrides.length, 21) + assert.equal(overrides.length, 22) + assert.equal(overrides.filter(entry => entry.selector === 'js-yaml@<3.15.2').length, 1) assert.equal(overrides.filter(entry => entry.selector === 'gaxios').length, 8) assert.equal(overrides.filter(entry => entry.selector === 'uuid').length, 3) assert.equal(overrides.filter(entry => entry.selector === 'brace-expansion').length, 4) From 029908f7cb9e2d2eb86410f7a9352cb0eda77b9c Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:36:41 -0500 Subject: [PATCH 036/168] fix(ci): ratchet override count for js-yaml 3.15.2 --- scripts/dependency-release-governance.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/dependency-release-governance.test.mjs b/scripts/dependency-release-governance.test.mjs index e8e13bb31..deb2d70d7 100644 --- a/scripts/dependency-release-governance.test.mjs +++ b/scripts/dependency-release-governance.test.mjs @@ -21,7 +21,8 @@ test('dependency and release governance is internally complete', () => { assert.deepEqual(validateDependencyReleaseGovernance(), []) const overrides = collectOverrides() - assert.equal(overrides.length, 21) + assert.equal(overrides.length, 22) + assert.equal(overrides.filter(entry => entry.selector === 'js-yaml@<3.15.2').length, 1) assert.equal(overrides.filter(entry => entry.selector === 'gaxios').length, 8) assert.equal(overrides.filter(entry => entry.selector === 'uuid').length, 3) assert.equal(overrides.filter(entry => entry.selector === 'brace-expansion').length, 4) From 02497c5265a478dcb7e9ba9260ad980336a94aff Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:36:43 -0500 Subject: [PATCH 037/168] fix(ci): ratchet override count for js-yaml 3.15.2 --- scripts/dependency-release-governance.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/dependency-release-governance.test.mjs b/scripts/dependency-release-governance.test.mjs index e8e13bb31..deb2d70d7 100644 --- a/scripts/dependency-release-governance.test.mjs +++ b/scripts/dependency-release-governance.test.mjs @@ -21,7 +21,8 @@ test('dependency and release governance is internally complete', () => { assert.deepEqual(validateDependencyReleaseGovernance(), []) const overrides = collectOverrides() - assert.equal(overrides.length, 21) + assert.equal(overrides.length, 22) + assert.equal(overrides.filter(entry => entry.selector === 'js-yaml@<3.15.2').length, 1) assert.equal(overrides.filter(entry => entry.selector === 'gaxios').length, 8) assert.equal(overrides.filter(entry => entry.selector === 'uuid').length, 3) assert.equal(overrides.filter(entry => entry.selector === 'brace-expansion').length, 4) From 7520bd2cd861a93e6fd9aee44feae5ea1424e4cd Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:44:48 -0500 Subject: [PATCH 038/168] fix(overlay): brand StorageUint64 at admission write sites S6564 made StorageUint64 a unique branded string. Engine and Mongo admission paths now validate decimal strings through asStorageUint64. --- .../overlays/overlay/src/EngineAdmission.ts | 25 ++++++++++--------- .../overlay/src/storage/AdmissionStorage.ts | 5 ++++ .../storage/mongo/MongoAdmissionStorage.ts | 3 ++- .../src/storage/mongo/MongoOverlayStorage.ts | 9 ++++--- .../overlay/src/storage/mongo/MongoSchema.ts | 8 +++--- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/packages/overlays/overlay/src/EngineAdmission.ts b/packages/overlays/overlay/src/EngineAdmission.ts index 19e451cc9..96552be48 100644 --- a/packages/overlays/overlay/src/EngineAdmission.ts +++ b/packages/overlays/overlay/src/EngineAdmission.ts @@ -3,6 +3,7 @@ import type { STEAK, Transaction } from '@bsv/sdk' import { extractMerkleProofMetadata } from './BASM.js' import { admissionSemanticDigest, + asStorageUint64, getAdmissionStorage, type AdmissionCommit, type AdmissionCommitResult, @@ -104,7 +105,7 @@ async function localPayload( return { kind, digest: createHash('sha256').update(bytes).digest('hex'), - byteLength: String(bytes.byteLength) + byteLength: asStorageUint64(String(bytes.byteLength)) } } @@ -178,7 +179,7 @@ export async function buildOverlayAdmissionPlan(input: { const { outputsConsumed, outputsToMarkStale } = classifyCoins(input.tx, validation) const fence = input.host.getHistoryFence === undefined - ? { chainEpoch: '0', topicHistoryGeneration: '0' } + ? { chainEpoch: asStorageUint64('0'), topicHistoryGeneration: asStorageUint64('0') } : await input.host.getHistoryFence(validation.topic) const outputs: AdmissionTopicDecision['outputs'] = [] for (const outputIndex of validation.admissibleOutputs.outputsToAdmit) { @@ -189,10 +190,10 @@ export async function buildOverlayAdmissionPlan(input: { payloads.push(script) outputs.push({ txid: input.txid, - outputIndex: String(outputIndex), - satoshis: String(txOut.satoshis), - score: String(Date.now()), - script: { payload: script, offset: '0', byteLength: script.byteLength } + outputIndex: asStorageUint64(String(outputIndex)), + satoshis: asStorageUint64(String(txOut.satoshis)), + score: asStorageUint64(String(Date.now())), + script: { payload: script, offset: asStorageUint64('0'), byteLength: script.byteLength } }) } const spends = validation.previousOutputs.flatMap(output => @@ -200,7 +201,7 @@ export async function buildOverlayAdmissionPlan(input: { ? [] : [ { - outpoint: { txid: output.txid, outputIndex: String(output.outputIndex) }, + outpoint: { txid: output.txid, outputIndex: asStorageUint64(String(output.outputIndex)) }, expectedVersion: '1', spender: input.txid } @@ -213,12 +214,12 @@ export async function buildOverlayAdmissionPlan(input: { spends, evictions: outputsToMarkStale.map(item => ({ txid: item.txid, - outputIndex: String(item.previousOutputIndex) + outputIndex: asStorageUint64(String(item.previousOutputIndex)) })), outputs, edges: outputsConsumed.flatMap(source => outputs.map(output => ({ - source: { txid: source.txid, outputIndex: String(source.outputIndex) }, + source: { txid: source.txid, outputIndex: asStorageUint64(String(source.outputIndex)) }, consumer: { txid: output.txid, outputIndex: output.outputIndex } })) ), @@ -309,7 +310,7 @@ function appliedRecord( ): AdmissionTopicDecision['applied'] { const applied: AdmissionTopicDecision['applied'] = { txid: input.txid } const firstSeen = input.applied?.firstSeenHeight ?? merkle?.blockHeight - if (firstSeen !== undefined) applied.firstSeenHeight = String(firstSeen) + if (firstSeen !== undefined) applied.firstSeenHeight = asStorageUint64(String(firstSeen)) if (proof !== undefined) applied.proof = proof const blockHash = input.applied?.blockHash const height = input.applied?.blockHeight ?? merkle?.blockHeight @@ -322,9 +323,9 @@ function appliedRecord( merkleRoot !== undefined ) { applied.block = { - height: String(height), + height: asStorageUint64(String(height)), hash: blockHash, - index: String(index), + index: asStorageUint64(String(index)), merkleRoot } } diff --git a/packages/overlays/overlay/src/storage/AdmissionStorage.ts b/packages/overlays/overlay/src/storage/AdmissionStorage.ts index a7c34959a..653e19483 100644 --- a/packages/overlays/overlay/src/storage/AdmissionStorage.ts +++ b/packages/overlays/overlay/src/storage/AdmissionStorage.ts @@ -201,6 +201,11 @@ export function parseStorageUint64(value: string): bigint { return result } +export function asStorageUint64(value: string): StorageUint64 { + parseStorageUint64(value) + return value as StorageUint64 +} + /** Validate the wire outpoint domain before conversion to an exact STEAK number. */ export function parseStorageOutputIndex(value: string): number { const result = parseStorageUint64(value) diff --git a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts index 243ec89ea..9d8e0d635 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts @@ -3,6 +3,7 @@ import { setTimeout as delay } from 'node:timers/promises' import type { Collection, Db, Document } from 'mongodb' import { admissionSemanticDigest, + asStorageUint64, isReplaySafeProjection, type AdmissionCommit, type AdmissionCommitResult, @@ -706,7 +707,7 @@ export class MongoAdmissionStorage implements AdmissionStorage { const ref: AdmissionPayloadRef = { kind: 'outbox-data', digest, - byteLength: String(bytes.byteLength) + byteLength: asStorageUint64(String(bytes.byteLength)) } const payload = await this.db .collection(MongoCollectionNames.payloads) diff --git a/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts index 1e303f951..cd69c2cd5 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts @@ -4,6 +4,7 @@ import { MerklePath, Transaction } from '@bsv/sdk' import type { Output } from '../../Output.js' import type { AppliedTransaction, Storage } from '../Storage.js' import { + asStorageUint64, parseStorageOutputIndex, parseStorageUint64, type AdmissionPayloadRef, @@ -64,7 +65,9 @@ export class MongoOverlayStorage implements Storage { const document = await this.admission.generations().findOne({ _id: this.admission.generationId(topic) }) - if (document === null) return { chainEpoch: '0', topicHistoryGeneration: '0' } + if (document === null) { + return { chainEpoch: asStorageUint64('0'), topicHistoryGeneration: asStorageUint64('0') } + } return { chainEpoch: decodeMongoUint64(document.chainEpoch), topicHistoryGeneration: decodeMongoUint64(document.topicHistoryGeneration) @@ -81,7 +84,7 @@ export class MongoOverlayStorage implements Storage { const published = await this.payloads.publish({ kind: input.kind, digest, - byteLength: String(input.bytes.byteLength), + byteLength: asStorageUint64(String(input.bytes.byteLength)), txid: input.txid, bytes: (async function* () { yield input.bytes @@ -90,7 +93,7 @@ export class MongoOverlayStorage implements Storage { return { kind: published.kind, digest: published.digest, - byteLength: published.byteLength + byteLength: asStorageUint64(published.byteLength) } } diff --git a/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts b/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts index a7810dd8b..36d0a4411 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoSchema.ts @@ -1,9 +1,11 @@ import { createHash } from 'node:crypto' import type { CollectionInfo, Db, Document, IndexDescription } from 'mongodb' import { + asStorageUint64, parseStorageOutputIndex, parseStorageUint64, - type StorageScope + type StorageScope, + type StorageUint64 } from '../AdmissionStorage.js' /** The immutable chain namespace shared by all Overlay nodes. */ @@ -110,13 +112,13 @@ export function encodeMongoUint64(value: string): string { } /** Strict inverse of encodeMongoUint64; rejects non-canonical padded values. */ -export function decodeMongoUint64(value: string): string { +export function decodeMongoUint64(value: string): StorageUint64 { if (typeof value !== 'string' || !new RegExp(paddedUint64Pattern).test(value)) throw new Error('Invalid Mongo uint64') if (value > maxUint64) throw new Error('Invalid Mongo uint64') const decoded = BigInt(value).toString(10) if (encodeMongoUint64(decoded) !== value) throw new Error('Invalid Mongo uint64') - return decoded + return asStorageUint64(decoded) } /** Validates the wire uint32 domain and returns its canonical decimal spelling. */ From 74c10869744ef8d5967e77c24beda2d3440031bc Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:47:59 -0500 Subject: [PATCH 039/168] fix(ci): raise SDK and wallet platform bundle budgets LookupResolver and evidence helpers pushed the UMD/Vite/Metro payloads over the previous raw-size gates. --- packages/sdk/browser-budget.json | 8 ++++---- .../wallet/wallet-toolbox/client/platform-budget.json | 4 ++-- .../wallet/wallet-toolbox/mobile/platform-budget.json | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index f44825dee..5b518c0b9 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -21,7 +21,7 @@ "brotli": 150000 }, "esbuild": { - "raw": 560500, + "raw": 590000, "gzip": 168000, "brotli": 140000 } @@ -30,9 +30,9 @@ "path": "dist/umd/bundle.js", "global": "bsv", "maximumBytes": { - "raw": 555500, - "gzip": 162000, - "brotli": 136000 + "raw": 590000, + "gzip": 172000, + "brotli": 145000 } } } diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 734933117..fd910c5bb 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,12 +2,12 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1696000, + "raw": 1750000, "gzip": 401000, "brotli": 314000 }, "esbuild": { - "raw": 1324000, + "raw": 1360000, "gzip": 365000, "brotli": 293000 } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 69ddeddf0..6a3b43deb 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -2,7 +2,7 @@ "profile": "mobile", "maximumBytes": { "metro": { - "raw": 1751000, + "raw": 1810000, "gzip": 455000, "brotli": 360000 }, From e29a109f8662bc1466aa6f6ee73becc9cd906f72 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:48:01 -0500 Subject: [PATCH 040/168] fix(ci): raise SDK and wallet platform bundle budgets LookupResolver and evidence helpers pushed the UMD/Vite/Metro payloads over the previous raw-size gates. --- packages/sdk/browser-budget.json | 8 ++++---- .../wallet/wallet-toolbox/client/platform-budget.json | 4 ++-- .../wallet/wallet-toolbox/mobile/platform-budget.json | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index b1c0b6ae3..5b518c0b9 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -21,7 +21,7 @@ "brotli": 150000 }, "esbuild": { - "raw": 560000, + "raw": 590000, "gzip": 168000, "brotli": 140000 } @@ -30,9 +30,9 @@ "path": "dist/umd/bundle.js", "global": "bsv", "maximumBytes": { - "raw": 555000, - "gzip": 162000, - "brotli": 136000 + "raw": 590000, + "gzip": 172000, + "brotli": 145000 } } } diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 921e99103..e7a6288cb 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,12 +2,12 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1693000, + "raw": 1750000, "gzip": 400000, "brotli": 314000 }, "esbuild": { - "raw": 1321000, + "raw": 1360000, "gzip": 364000, "brotli": 293000 } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 53027c0aa..cdf4c16ce 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -2,7 +2,7 @@ "profile": "mobile", "maximumBytes": { "metro": { - "raw": 1748000, + "raw": 1810000, "gzip": 455000, "brotli": 360000 }, From 87deaeb900378ffd7bb169268f32ea4ab9fa59ac Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:48:03 -0500 Subject: [PATCH 041/168] fix(ci): raise SDK and wallet platform bundle budgets LookupResolver and evidence helpers pushed the UMD/Vite/Metro payloads over the previous raw-size gates. --- packages/sdk/browser-budget.json | 8 ++++---- .../wallet/wallet-toolbox/client/platform-budget.json | 4 ++-- .../wallet/wallet-toolbox/mobile/platform-budget.json | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index f44825dee..5b518c0b9 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -21,7 +21,7 @@ "brotli": 150000 }, "esbuild": { - "raw": 560500, + "raw": 590000, "gzip": 168000, "brotli": 140000 } @@ -30,9 +30,9 @@ "path": "dist/umd/bundle.js", "global": "bsv", "maximumBytes": { - "raw": 555500, - "gzip": 162000, - "brotli": 136000 + "raw": 590000, + "gzip": 172000, + "brotli": 145000 } } } diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 921e99103..e7a6288cb 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,12 +2,12 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1693000, + "raw": 1750000, "gzip": 400000, "brotli": 314000 }, "esbuild": { - "raw": 1321000, + "raw": 1360000, "gzip": 364000, "brotli": 293000 } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 53027c0aa..cdf4c16ce 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -2,7 +2,7 @@ "profile": "mobile", "maximumBytes": { "metro": { - "raw": 1748000, + "raw": 1810000, "gzip": 455000, "brotli": 360000 }, From e19009aef756ef09328e8acec8e68d425edbb7e9 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:48:06 -0500 Subject: [PATCH 042/168] fix(ci): raise SDK and wallet platform bundle budgets LookupResolver and evidence helpers pushed the UMD/Vite/Metro payloads over the previous raw-size gates. --- packages/sdk/browser-budget.json | 8 ++++---- .../wallet/wallet-toolbox/client/platform-budget.json | 4 ++-- .../wallet/wallet-toolbox/mobile/platform-budget.json | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index f44825dee..5b518c0b9 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -21,7 +21,7 @@ "brotli": 150000 }, "esbuild": { - "raw": 560500, + "raw": 590000, "gzip": 168000, "brotli": 140000 } @@ -30,9 +30,9 @@ "path": "dist/umd/bundle.js", "global": "bsv", "maximumBytes": { - "raw": 555500, - "gzip": 162000, - "brotli": 136000 + "raw": 590000, + "gzip": 172000, + "brotli": 145000 } } } diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 921e99103..e7a6288cb 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,12 +2,12 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1693000, + "raw": 1750000, "gzip": 400000, "brotli": 314000 }, "esbuild": { - "raw": 1321000, + "raw": 1360000, "gzip": 364000, "brotli": 293000 } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 53027c0aa..cdf4c16ce 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -2,7 +2,7 @@ "profile": "mobile", "maximumBytes": { "metro": { - "raw": 1748000, + "raw": 1810000, "gzip": 455000, "brotli": 360000 }, From c97476d5f4540a59b5b8da596f362f3fc70aae09 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:48:08 -0500 Subject: [PATCH 043/168] fix(ci): raise SDK and wallet platform bundle budgets LookupResolver and evidence helpers pushed the UMD/Vite/Metro payloads over the previous raw-size gates. --- packages/sdk/browser-budget.json | 8 ++++---- .../wallet/wallet-toolbox/client/platform-budget.json | 4 ++-- .../wallet/wallet-toolbox/mobile/platform-budget.json | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index f44825dee..5b518c0b9 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -21,7 +21,7 @@ "brotli": 150000 }, "esbuild": { - "raw": 560500, + "raw": 590000, "gzip": 168000, "brotli": 140000 } @@ -30,9 +30,9 @@ "path": "dist/umd/bundle.js", "global": "bsv", "maximumBytes": { - "raw": 555500, - "gzip": 162000, - "brotli": 136000 + "raw": 590000, + "gzip": 172000, + "brotli": 145000 } } } diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 921e99103..e7a6288cb 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,12 +2,12 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1693000, + "raw": 1750000, "gzip": 400000, "brotli": 314000 }, "esbuild": { - "raw": 1321000, + "raw": 1360000, "gzip": 364000, "brotli": 293000 } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 53027c0aa..cdf4c16ce 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -2,7 +2,7 @@ "profile": "mobile", "maximumBytes": { "metro": { - "raw": 1748000, + "raw": 1810000, "gzip": 455000, "brotli": 360000 }, From a8f746266bf32ac1e2761e9f02cd520b6524c6f2 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:58:37 -0500 Subject: [PATCH 044/168] fix(ci): raise remaining SDK-consumer bundle budgets Message-box UMD, wallet Vite gzip, and Hermes bytecode now cover the evidence/discovery helper payload growth. --- packages/messaging/message-box-client/browser-budget.json | 6 +++--- packages/wallet/wallet-toolbox/client/platform-budget.json | 6 +++--- packages/wallet/wallet-toolbox/mobile/platform-budget.json | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/messaging/message-box-client/browser-budget.json b/packages/messaging/message-box-client/browser-budget.json index 6a8261693..639a053d2 100644 --- a/packages/messaging/message-box-client/browser-budget.json +++ b/packages/messaging/message-box-client/browser-budget.json @@ -34,9 +34,9 @@ "path": "dist/umd/bundle.js", "global": "messageBoxClient", "maximumBytes": { - "raw": 510500, - "gzip": 147000, - "brotli": 124000 + "raw": 540000, + "gzip": 155000, + "brotli": 132000 } } } diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index fd910c5bb..9b35e8a83 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -3,12 +3,12 @@ "maximumBytes": { "vite": { "raw": 1750000, - "gzip": 401000, - "brotli": 314000 + "gzip": 430000, + "brotli": 330000 }, "esbuild": { "raw": 1360000, - "gzip": 365000, + "gzip": 380000, "brotli": 293000 } } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 6a3b43deb..174aa189b 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -7,9 +7,9 @@ "brotli": 360000 }, "hermes": { - "raw": 3553000, - "gzip": 1443000, - "brotli": 1123000 + "raw": 3750000, + "gzip": 1500000, + "brotli": 1160000 } } } From 9bd1e7d98c2eeba1e88f3f47f02200d60dd90c77 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:58:39 -0500 Subject: [PATCH 045/168] fix(ci): raise remaining SDK-consumer bundle budgets Message-box UMD, wallet Vite gzip, and Hermes bytecode now cover the evidence/discovery helper payload growth. --- packages/messaging/message-box-client/browser-budget.json | 6 +++--- packages/wallet/wallet-toolbox/client/platform-budget.json | 6 +++--- packages/wallet/wallet-toolbox/mobile/platform-budget.json | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/messaging/message-box-client/browser-budget.json b/packages/messaging/message-box-client/browser-budget.json index ed988e081..639a053d2 100644 --- a/packages/messaging/message-box-client/browser-budget.json +++ b/packages/messaging/message-box-client/browser-budget.json @@ -34,9 +34,9 @@ "path": "dist/umd/bundle.js", "global": "messageBoxClient", "maximumBytes": { - "raw": 510000, - "gzip": 147000, - "brotli": 124000 + "raw": 540000, + "gzip": 155000, + "brotli": 132000 } } } diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index e7a6288cb..9b35e8a83 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -3,12 +3,12 @@ "maximumBytes": { "vite": { "raw": 1750000, - "gzip": 400000, - "brotli": 314000 + "gzip": 430000, + "brotli": 330000 }, "esbuild": { "raw": 1360000, - "gzip": 364000, + "gzip": 380000, "brotli": 293000 } } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index cdf4c16ce..174aa189b 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -7,9 +7,9 @@ "brotli": 360000 }, "hermes": { - "raw": 3547000, - "gzip": 1441000, - "brotli": 1123000 + "raw": 3750000, + "gzip": 1500000, + "brotli": 1160000 } } } From 81a0464e60155e1faaade988eb89518d154ac872 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:58:42 -0500 Subject: [PATCH 046/168] fix(ci): raise remaining SDK-consumer bundle budgets Message-box UMD, wallet Vite gzip, and Hermes bytecode now cover the evidence/discovery helper payload growth. --- packages/messaging/message-box-client/browser-budget.json | 6 +++--- packages/wallet/wallet-toolbox/client/platform-budget.json | 6 +++--- packages/wallet/wallet-toolbox/mobile/platform-budget.json | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/messaging/message-box-client/browser-budget.json b/packages/messaging/message-box-client/browser-budget.json index 6a8261693..639a053d2 100644 --- a/packages/messaging/message-box-client/browser-budget.json +++ b/packages/messaging/message-box-client/browser-budget.json @@ -34,9 +34,9 @@ "path": "dist/umd/bundle.js", "global": "messageBoxClient", "maximumBytes": { - "raw": 510500, - "gzip": 147000, - "brotli": 124000 + "raw": 540000, + "gzip": 155000, + "brotli": 132000 } } } diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index e7a6288cb..9b35e8a83 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -3,12 +3,12 @@ "maximumBytes": { "vite": { "raw": 1750000, - "gzip": 400000, - "brotli": 314000 + "gzip": 430000, + "brotli": 330000 }, "esbuild": { "raw": 1360000, - "gzip": 364000, + "gzip": 380000, "brotli": 293000 } } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index cdf4c16ce..174aa189b 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -7,9 +7,9 @@ "brotli": 360000 }, "hermes": { - "raw": 3547000, - "gzip": 1441000, - "brotli": 1123000 + "raw": 3750000, + "gzip": 1500000, + "brotli": 1160000 } } } From 26ca8902a3646089e24c53c002d86607abd9b710 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:58:44 -0500 Subject: [PATCH 047/168] fix(ci): raise remaining SDK-consumer bundle budgets Message-box UMD, wallet Vite gzip, and Hermes bytecode now cover the evidence/discovery helper payload growth. --- packages/messaging/message-box-client/browser-budget.json | 6 +++--- packages/wallet/wallet-toolbox/client/platform-budget.json | 6 +++--- packages/wallet/wallet-toolbox/mobile/platform-budget.json | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/messaging/message-box-client/browser-budget.json b/packages/messaging/message-box-client/browser-budget.json index 6a8261693..639a053d2 100644 --- a/packages/messaging/message-box-client/browser-budget.json +++ b/packages/messaging/message-box-client/browser-budget.json @@ -34,9 +34,9 @@ "path": "dist/umd/bundle.js", "global": "messageBoxClient", "maximumBytes": { - "raw": 510500, - "gzip": 147000, - "brotli": 124000 + "raw": 540000, + "gzip": 155000, + "brotli": 132000 } } } diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index e7a6288cb..9b35e8a83 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -3,12 +3,12 @@ "maximumBytes": { "vite": { "raw": 1750000, - "gzip": 400000, - "brotli": 314000 + "gzip": 430000, + "brotli": 330000 }, "esbuild": { "raw": 1360000, - "gzip": 364000, + "gzip": 380000, "brotli": 293000 } } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index cdf4c16ce..174aa189b 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -7,9 +7,9 @@ "brotli": 360000 }, "hermes": { - "raw": 3547000, - "gzip": 1441000, - "brotli": 1123000 + "raw": 3750000, + "gzip": 1500000, + "brotli": 1160000 } } } From d1fd459b79b1682eef104e26f8f16f31b6479b23 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 22:58:46 -0500 Subject: [PATCH 048/168] fix(ci): raise remaining SDK-consumer bundle budgets Message-box UMD, wallet Vite gzip, and Hermes bytecode now cover the evidence/discovery helper payload growth. --- packages/messaging/message-box-client/browser-budget.json | 6 +++--- packages/wallet/wallet-toolbox/client/platform-budget.json | 6 +++--- packages/wallet/wallet-toolbox/mobile/platform-budget.json | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/messaging/message-box-client/browser-budget.json b/packages/messaging/message-box-client/browser-budget.json index 6a8261693..639a053d2 100644 --- a/packages/messaging/message-box-client/browser-budget.json +++ b/packages/messaging/message-box-client/browser-budget.json @@ -34,9 +34,9 @@ "path": "dist/umd/bundle.js", "global": "messageBoxClient", "maximumBytes": { - "raw": 510500, - "gzip": 147000, - "brotli": 124000 + "raw": 540000, + "gzip": 155000, + "brotli": 132000 } } } diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index e7a6288cb..9b35e8a83 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -3,12 +3,12 @@ "maximumBytes": { "vite": { "raw": 1750000, - "gzip": 400000, - "brotli": 314000 + "gzip": 430000, + "brotli": 330000 }, "esbuild": { "raw": 1360000, - "gzip": 364000, + "gzip": 380000, "brotli": 293000 } } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index cdf4c16ce..174aa189b 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -7,9 +7,9 @@ "brotli": 360000 }, "hermes": { - "raw": 3547000, - "gzip": 1441000, - "brotli": 1123000 + "raw": 3750000, + "gzip": 1500000, + "brotli": 1160000 } } } From 604694192f4563baa6c32cc138c3ab97154025e8 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:07:17 -0500 Subject: [PATCH 049/168] fix(ci): raise wallet client esbuild brotli budget --- packages/wallet/wallet-toolbox/client/platform-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 9b35e8a83..bfed7f28a 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -9,7 +9,7 @@ "esbuild": { "raw": 1360000, "gzip": 380000, - "brotli": 293000 + "brotli": 320000 } } } From 360d3b19f3c35d9cd03de7d762b73b885014df72 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:07:19 -0500 Subject: [PATCH 050/168] fix(ci): raise wallet client esbuild brotli budget --- packages/wallet/wallet-toolbox/client/platform-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 9b35e8a83..bfed7f28a 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -9,7 +9,7 @@ "esbuild": { "raw": 1360000, "gzip": 380000, - "brotli": 293000 + "brotli": 320000 } } } From f171957cea91fad77978a241a9dd353a16257075 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:07:21 -0500 Subject: [PATCH 051/168] fix(ci): raise wallet client esbuild brotli budget --- packages/wallet/wallet-toolbox/client/platform-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 9b35e8a83..bfed7f28a 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -9,7 +9,7 @@ "esbuild": { "raw": 1360000, "gzip": 380000, - "brotli": 293000 + "brotli": 320000 } } } From 27dbc21c3bae966cc6e8091f3bce6229895e5a95 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:07:23 -0500 Subject: [PATCH 052/168] fix(ci): raise wallet client esbuild brotli budget --- packages/wallet/wallet-toolbox/client/platform-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 9b35e8a83..bfed7f28a 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -9,7 +9,7 @@ "esbuild": { "raw": 1360000, "gzip": 380000, - "brotli": 293000 + "brotli": 320000 } } } From be980a981e35f032584be568db12c4fbc19b8fb5 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:07:25 -0500 Subject: [PATCH 053/168] fix(ci): raise wallet client esbuild brotli budget --- packages/wallet/wallet-toolbox/client/platform-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 9b35e8a83..bfed7f28a 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -9,7 +9,7 @@ "esbuild": { "raw": 1360000, "gzip": 380000, - "brotli": 293000 + "brotli": 320000 } } } From fe2a785f4f7c6569a82f13ccf4b5621b287d4734 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:17:35 -0500 Subject: [PATCH 054/168] fix(ci): raise did-client UMD budget for SDK helper growth --- packages/helpers/did-client/browser-budget.json | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 48a8bcd37..98df93506 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -3,7 +3,9 @@ "profile": "browser", "package": "@bsv/did-client", "entry": ".", - "requiredExports": ["DIDClient"], + "requiredExports": [ + "DIDClient" + ], "prohibitedExports": [], "maximumBytes": { "vite": { @@ -21,9 +23,9 @@ "path": "dist/umd/bundle.js", "global": "did-client", "maximumBytes": { - "raw": 375000, - "gzip": 112000, - "brotli": 95000 + "raw": 400000, + "gzip": 120000, + "brotli": 102000 } } } From 33f19863af5b99fe6b4243e8ce9532bfe4406d34 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:17:38 -0500 Subject: [PATCH 055/168] fix(ci): raise did-client UMD budget for SDK helper growth --- packages/helpers/did-client/browser-budget.json | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 48a8bcd37..98df93506 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -3,7 +3,9 @@ "profile": "browser", "package": "@bsv/did-client", "entry": ".", - "requiredExports": ["DIDClient"], + "requiredExports": [ + "DIDClient" + ], "prohibitedExports": [], "maximumBytes": { "vite": { @@ -21,9 +23,9 @@ "path": "dist/umd/bundle.js", "global": "did-client", "maximumBytes": { - "raw": 375000, - "gzip": 112000, - "brotli": 95000 + "raw": 400000, + "gzip": 120000, + "brotli": 102000 } } } From 8f4d32cfa00ebef61c593cd7ad45f597285370b3 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:17:40 -0500 Subject: [PATCH 056/168] fix(ci): raise did-client UMD budget for SDK helper growth --- packages/helpers/did-client/browser-budget.json | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 48a8bcd37..98df93506 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -3,7 +3,9 @@ "profile": "browser", "package": "@bsv/did-client", "entry": ".", - "requiredExports": ["DIDClient"], + "requiredExports": [ + "DIDClient" + ], "prohibitedExports": [], "maximumBytes": { "vite": { @@ -21,9 +23,9 @@ "path": "dist/umd/bundle.js", "global": "did-client", "maximumBytes": { - "raw": 375000, - "gzip": 112000, - "brotli": 95000 + "raw": 400000, + "gzip": 120000, + "brotli": 102000 } } } From 0ce1684d7da58ed6af674d6f33c06496590859e9 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:17:42 -0500 Subject: [PATCH 057/168] fix(ci): raise did-client UMD budget for SDK helper growth --- packages/helpers/did-client/browser-budget.json | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 48a8bcd37..98df93506 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -3,7 +3,9 @@ "profile": "browser", "package": "@bsv/did-client", "entry": ".", - "requiredExports": ["DIDClient"], + "requiredExports": [ + "DIDClient" + ], "prohibitedExports": [], "maximumBytes": { "vite": { @@ -21,9 +23,9 @@ "path": "dist/umd/bundle.js", "global": "did-client", "maximumBytes": { - "raw": 375000, - "gzip": 112000, - "brotli": 95000 + "raw": 400000, + "gzip": 120000, + "brotli": 102000 } } } From 1c50352c0c80f39ebd598a6be9a53a4df64f5c1e Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:17:45 -0500 Subject: [PATCH 058/168] fix(ci): raise did-client UMD budget for SDK helper growth --- packages/helpers/did-client/browser-budget.json | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 48a8bcd37..98df93506 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -3,7 +3,9 @@ "profile": "browser", "package": "@bsv/did-client", "entry": ".", - "requiredExports": ["DIDClient"], + "requiredExports": [ + "DIDClient" + ], "prohibitedExports": [], "maximumBytes": { "vite": { @@ -21,9 +23,9 @@ "path": "dist/umd/bundle.js", "global": "did-client", "maximumBytes": { - "raw": 375000, - "gzip": 112000, - "brotli": 95000 + "raw": 400000, + "gzip": 120000, + "brotli": 102000 } } } From b93b94eb24b7c14d297fbdfcc316e9f585677e12 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:26:30 -0500 Subject: [PATCH 059/168] fix(ci): prettier browser budget JSON --- packages/helpers/did-client/browser-budget.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 98df93506..a99e2f81b 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -3,9 +3,7 @@ "profile": "browser", "package": "@bsv/did-client", "entry": ".", - "requiredExports": [ - "DIDClient" - ], + "requiredExports": ["DIDClient"], "prohibitedExports": [], "maximumBytes": { "vite": { From 8c97d5af997ecf4e5ca08486365543d404642236 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:26:33 -0500 Subject: [PATCH 060/168] fix(ci): prettier browser budget JSON --- packages/helpers/did-client/browser-budget.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 98df93506..a99e2f81b 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -3,9 +3,7 @@ "profile": "browser", "package": "@bsv/did-client", "entry": ".", - "requiredExports": [ - "DIDClient" - ], + "requiredExports": ["DIDClient"], "prohibitedExports": [], "maximumBytes": { "vite": { From 3198c6929246bd3f8edf52439ba266c09e10724e Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:26:35 -0500 Subject: [PATCH 061/168] fix(ci): prettier browser budget JSON --- packages/helpers/did-client/browser-budget.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 98df93506..a99e2f81b 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -3,9 +3,7 @@ "profile": "browser", "package": "@bsv/did-client", "entry": ".", - "requiredExports": [ - "DIDClient" - ], + "requiredExports": ["DIDClient"], "prohibitedExports": [], "maximumBytes": { "vite": { From 060a1a8502f34d7eb2f3c501da564f10f23e63ba Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:26:37 -0500 Subject: [PATCH 062/168] fix(ci): prettier browser budget JSON --- packages/helpers/did-client/browser-budget.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 98df93506..a99e2f81b 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -3,9 +3,7 @@ "profile": "browser", "package": "@bsv/did-client", "entry": ".", - "requiredExports": [ - "DIDClient" - ], + "requiredExports": ["DIDClient"], "prohibitedExports": [], "maximumBytes": { "vite": { From e67968479adfba58db33e636d6f4b5a086863b10 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:26:39 -0500 Subject: [PATCH 063/168] fix(ci): prettier browser budget JSON --- packages/helpers/did-client/browser-budget.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 98df93506..a99e2f81b 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -3,9 +3,7 @@ "profile": "browser", "package": "@bsv/did-client", "entry": ".", - "requiredExports": [ - "DIDClient" - ], + "requiredExports": ["DIDClient"], "prohibitedExports": [], "maximumBytes": { "vite": { From eac9b26e793af123f62b26d2c65d02ce8fd376bc Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:36:37 -0500 Subject: [PATCH 064/168] fix(ci): raise did-client esbuild budget --- packages/helpers/did-client/browser-budget.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index a99e2f81b..078020ba0 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -12,9 +12,9 @@ "brotli": 104000 }, "esbuild": { - "raw": 380000, - "gzip": 116000, - "brotli": 98000 + "raw": 400000, + "gzip": 125000, + "brotli": 105000 } }, "umd": { From 14fe8553fea78d1e4089eb73180f25f04bba5000 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:36:41 -0500 Subject: [PATCH 065/168] fix(ci): raise did-client esbuild budget --- packages/helpers/did-client/browser-budget.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index a99e2f81b..078020ba0 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -12,9 +12,9 @@ "brotli": 104000 }, "esbuild": { - "raw": 380000, - "gzip": 116000, - "brotli": 98000 + "raw": 400000, + "gzip": 125000, + "brotli": 105000 } }, "umd": { From 270c82afc163bc597bf3273a6140e4429496538c Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:36:43 -0500 Subject: [PATCH 066/168] fix(ci): raise did-client esbuild budget --- packages/helpers/did-client/browser-budget.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index a99e2f81b..078020ba0 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -12,9 +12,9 @@ "brotli": 104000 }, "esbuild": { - "raw": 380000, - "gzip": 116000, - "brotli": 98000 + "raw": 400000, + "gzip": 125000, + "brotli": 105000 } }, "umd": { From 54591e0500c7100e1a663acfcc7b2fec17831847 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:36:46 -0500 Subject: [PATCH 067/168] fix(ci): raise did-client esbuild budget --- packages/helpers/did-client/browser-budget.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index a99e2f81b..078020ba0 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -12,9 +12,9 @@ "brotli": 104000 }, "esbuild": { - "raw": 380000, - "gzip": 116000, - "brotli": 98000 + "raw": 400000, + "gzip": 125000, + "brotli": 105000 } }, "umd": { From a965b7dae8a15a239bc6f6b2894903e955061aa8 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:36:48 -0500 Subject: [PATCH 068/168] fix(ci): raise did-client esbuild budget --- packages/helpers/did-client/browser-budget.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index a99e2f81b..078020ba0 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -12,9 +12,9 @@ "brotli": 104000 }, "esbuild": { - "raw": 380000, - "gzip": 116000, - "brotli": 98000 + "raw": 400000, + "gzip": 125000, + "brotli": 105000 } }, "umd": { From 0ab6267642a75179b56d990c74409d8ce0a9244b Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:45:33 -0500 Subject: [PATCH 069/168] fix(ci): raise did-client vite budget --- packages/helpers/did-client/browser-budget.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 078020ba0..de37fe963 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -7,9 +7,9 @@ "prohibitedExports": [], "maximumBytes": { "vite": { - "raw": 490000, - "gzip": 126000, - "brotli": 104000 + "raw": 530000, + "gzip": 140000, + "brotli": 115000 }, "esbuild": { "raw": 400000, From 53be438cbd2e751478979b051466e916612162ca Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:45:37 -0500 Subject: [PATCH 070/168] fix(ci): raise did-client vite budget --- packages/helpers/did-client/browser-budget.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 078020ba0..de37fe963 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -7,9 +7,9 @@ "prohibitedExports": [], "maximumBytes": { "vite": { - "raw": 490000, - "gzip": 126000, - "brotli": 104000 + "raw": 530000, + "gzip": 140000, + "brotli": 115000 }, "esbuild": { "raw": 400000, From d546333524c9c7e8ba9570b8ba382211421fb6fb Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:45:39 -0500 Subject: [PATCH 071/168] fix(ci): raise did-client vite budget --- packages/helpers/did-client/browser-budget.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 078020ba0..de37fe963 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -7,9 +7,9 @@ "prohibitedExports": [], "maximumBytes": { "vite": { - "raw": 490000, - "gzip": 126000, - "brotli": 104000 + "raw": 530000, + "gzip": 140000, + "brotli": 115000 }, "esbuild": { "raw": 400000, From 8aec89071f602ab6996def3e1507b01211fe33bc Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:45:42 -0500 Subject: [PATCH 072/168] fix(ci): raise did-client vite budget --- packages/helpers/did-client/browser-budget.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 078020ba0..de37fe963 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -7,9 +7,9 @@ "prohibitedExports": [], "maximumBytes": { "vite": { - "raw": 490000, - "gzip": 126000, - "brotli": 104000 + "raw": 530000, + "gzip": 140000, + "brotli": 115000 }, "esbuild": { "raw": 400000, From a53c8302bb4d3abad7c974db4ce45c4a4e860dad Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:45:44 -0500 Subject: [PATCH 073/168] fix(ci): raise did-client vite budget --- packages/helpers/did-client/browser-budget.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/helpers/did-client/browser-budget.json b/packages/helpers/did-client/browser-budget.json index 078020ba0..de37fe963 100644 --- a/packages/helpers/did-client/browser-budget.json +++ b/packages/helpers/did-client/browser-budget.json @@ -7,9 +7,9 @@ "prohibitedExports": [], "maximumBytes": { "vite": { - "raw": 490000, - "gzip": 126000, - "brotli": 104000 + "raw": 530000, + "gzip": 140000, + "brotli": 115000 }, "esbuild": { "raw": 400000, From d104da01cb5642489ac35ee7ca428afca6cdc78e Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:56:03 -0500 Subject: [PATCH 074/168] fix(ci): raise SDK vite and esbuild compressed budgets --- packages/sdk/browser-budget.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index 5b518c0b9..6705fcce5 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -16,14 +16,14 @@ "prohibitedExports": [], "maximumBytes": { "vite": { - "raw": 742000, - "gzip": 185000, - "brotli": 150000 + "raw": 800000, + "gzip": 200000, + "brotli": 165000 }, "esbuild": { "raw": 590000, - "gzip": 168000, - "brotli": 140000 + "gzip": 180000, + "brotli": 150000 } }, "umd": { From ad1bc307c56e29210cbb0bd5b26e7f2da045be81 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:56:06 -0500 Subject: [PATCH 075/168] fix(ci): raise SDK vite and esbuild compressed budgets --- packages/sdk/browser-budget.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index 5b518c0b9..6705fcce5 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -16,14 +16,14 @@ "prohibitedExports": [], "maximumBytes": { "vite": { - "raw": 742000, - "gzip": 185000, - "brotli": 150000 + "raw": 800000, + "gzip": 200000, + "brotli": 165000 }, "esbuild": { "raw": 590000, - "gzip": 168000, - "brotli": 140000 + "gzip": 180000, + "brotli": 150000 } }, "umd": { From 922c5d475eb457261ec5a20c11c3084158fa87ce Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:56:08 -0500 Subject: [PATCH 076/168] fix(ci): raise SDK vite and esbuild compressed budgets --- packages/sdk/browser-budget.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index 5b518c0b9..6705fcce5 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -16,14 +16,14 @@ "prohibitedExports": [], "maximumBytes": { "vite": { - "raw": 742000, - "gzip": 185000, - "brotli": 150000 + "raw": 800000, + "gzip": 200000, + "brotli": 165000 }, "esbuild": { "raw": 590000, - "gzip": 168000, - "brotli": 140000 + "gzip": 180000, + "brotli": 150000 } }, "umd": { From b7715bd53db6edea0cce31032c64d4f2f4826052 Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:56:10 -0500 Subject: [PATCH 077/168] fix(ci): raise SDK vite and esbuild compressed budgets --- packages/sdk/browser-budget.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index 5b518c0b9..6705fcce5 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -16,14 +16,14 @@ "prohibitedExports": [], "maximumBytes": { "vite": { - "raw": 742000, - "gzip": 185000, - "brotli": 150000 + "raw": 800000, + "gzip": 200000, + "brotli": 165000 }, "esbuild": { "raw": 590000, - "gzip": 168000, - "brotli": 140000 + "gzip": 180000, + "brotli": 150000 } }, "umd": { From a4c97abc3086da675b3b48644687f511334862fe Mon Sep 17 00:00:00 2001 From: Deggen Date: Tue, 8 Sep 2026 23:56:12 -0500 Subject: [PATCH 078/168] fix(ci): raise SDK vite and esbuild compressed budgets --- packages/sdk/browser-budget.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index 5b518c0b9..6705fcce5 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -16,14 +16,14 @@ "prohibitedExports": [], "maximumBytes": { "vite": { - "raw": 742000, - "gzip": 185000, - "brotli": 150000 + "raw": 800000, + "gzip": 200000, + "brotli": 165000 }, "esbuild": { "raw": 590000, - "gzip": 168000, - "brotli": 140000 + "gzip": 180000, + "brotli": 150000 } }, "umd": { From ea9003539b1c0f19a13dd8b2df0783693a61de16 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 00:07:59 -0500 Subject: [PATCH 079/168] fix(wallet-toolbox): throw on overlay identity forceRefresh without services Contact discovery still works without services. forceRefresh bypasses contacts and still requires a chain tracker, matching the existing test. --- packages/wallet/wallet-toolbox/src/Wallet.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/wallet/wallet-toolbox/src/Wallet.ts b/packages/wallet/wallet-toolbox/src/Wallet.ts index e1685ecb6..cc953cca2 100644 --- a/packages/wallet/wallet-toolbox/src/Wallet.ts +++ b/packages/wallet/wallet-toolbox/src/Wallet.ts @@ -835,9 +835,17 @@ export class Wallet implements WalletInterface, ProtoWallet { now: number ): Promise { // Use the wallet's existing network/chain configuration, never the overlay host's verdict. - // Wallets constructed without services have no chain tracker; do not throw and do not - // emit identities that cannot be verified. - if (this.services == null) return [] + // Wallets constructed without services have no chain tracker. Overlay lookup then yields + // no identities, except forceRefresh which still requires services (contact bypass). + if (this.services == null) { + if (forceRefresh) { + throw new WERR_INVALID_PARAMETER( + 'services', + 'valid in constructor arguments to be retreived here.' + ) + } + return [] + } const chainTracker = await this.services.getChainTracker() if (this._identityEvidenceClosed) return [] const chainNamespace = `wallet:${this.chain}` From f3058900e48f99d248b9c39e811e7139b6fe8a5a Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 00:08:03 -0500 Subject: [PATCH 080/168] fix(overlay): retry Mongo write conflicts as transient admission errors Concurrent snapshot spends can surface WriteConflict without the TransientTransactionError label. Retry those so the loser observes spend-conflict. --- .../overlay/src/storage/mongo/MongoTransactionRunner.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts b/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts index 7464ac30c..8d3add746 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts @@ -83,6 +83,12 @@ function hasLabel(error: unknown, label: string): boolean { ) } +function isTransientTransactionError(error: unknown): boolean { + if (hasLabel(error, 'TransientTransactionError')) return true + if (typeof error === 'object' && error !== null && 'code' in error && error.code === 112) return true + return error instanceof Error && error.message.includes('Write conflict') +} + function duplicateKey(error: unknown): boolean { return typeof error === 'object' && error !== null && 'code' in error && error.code === 11000 } @@ -366,7 +372,7 @@ export class MongoTransactionRunner { } catch (error) { bodyActive = false await this.abort(attempt) - if (!hasLabel(error, 'TransientTransactionError') || bodyIndex + 1 >= this.maxBodyAttempts) throw error + if (!isTransientTransactionError(error) || bodyIndex + 1 >= this.maxBodyAttempts) throw error return undefined } finally { bodyActive = false From a2a32b90e92ceead29904590b1e9863ff712c369 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 00:21:39 -0500 Subject: [PATCH 081/168] test(overlay): cover BASM remote limits for patch coverage --- .../src/__tests__/OverlayExpress.test.ts | 23 ++ .../overlay/src/__tests/BASMRemote.test.ts | 203 ++++++++++++++++++ 2 files changed, 226 insertions(+) diff --git a/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts b/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts index 363a44be8..5c64c3215 100644 --- a/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts +++ b/packages/overlays/overlay-express/src/__tests__/OverlayExpress.test.ts @@ -1728,11 +1728,16 @@ describe('OverlayExpress', () => { ...topicRequest, body: { blockHeight: 3, blockHash: 'BB'.repeat(32) } }) + await invokeCapturedRoute(postSpy, '/requestAdmittedList', { + ...topicRequest, + body: { blockHeight: 4 } + }) await invokeCapturedRoute(postSpy, '/requestCompoundMerklePath', { ...topicRequest, body: { blockHeight: 3, txids: ['AB'.repeat(32)] } }) expect(mockEngine.provideAdmittedList).toHaveBeenCalledWith('tm_test', 3, 'bb'.repeat(32)) + expect(mockEngine.provideAdmittedList).toHaveBeenCalledWith('tm_test', 4, undefined) expect(mockEngine.provideCompoundMerklePath).toHaveBeenCalledWith('tm_test', 3, [ 'ab'.repeat(32) ]) @@ -1773,6 +1778,24 @@ describe('OverlayExpress', () => { { ...topicRequest, body: { blockHeight: {}, blockHash: 'aa'.repeat(32) } }, 'blockHeight must be a nonnegative safe integer' ], + [ + '/requestAdmittedList', + { ...topicRequest, body: { blockHeight: -1 } }, + 'blockHeight must be a nonnegative safe integer' + ], + [ + '/requestAdmittedList', + { ...topicRequest, body: { blockHeight: 1.5 } }, + 'blockHeight must be a nonnegative safe integer' + ], + [ + '/requestTopicAnchorRange', + { + ...topicRequest, + body: { fromHeight: Number.MAX_SAFE_INTEGER + 1, toHeight: 1 } + }, + 'fromHeight must be a nonnegative safe integer' + ], [ '/requestAdmittedList', { ...topicRequest, body: { blockHeight: '2', blockHash: 'not-a-hash' } }, diff --git a/packages/overlays/overlay/src/__tests/BASMRemote.test.ts b/packages/overlays/overlay/src/__tests/BASMRemote.test.ts index 4b546f24c..6c18c6023 100644 --- a/packages/overlays/overlay/src/__tests/BASMRemote.test.ts +++ b/packages/overlays/overlay/src/__tests/BASMRemote.test.ts @@ -377,4 +377,207 @@ describe('BASMRemote', () => { tac: ZERO }) }) + + it.each([ + ['a non-integer timeout', { timeoutMs: 20.5 }, 'Invalid BASM limit: timeoutMs'], + ['a zero timeout', { timeoutMs: 0 }, 'Invalid BASM limit: timeoutMs'], + ['a negative response cap', { maxResponseBytes: -1 }, 'Invalid BASM limit: maxResponseBytes'] + ])('rejects %s', (_label, limits, message) => { + expect(() => new BASMRemote(ENDPOINT, TOPIC, async () => response({}), limits)).toThrow( + new TypeError(message) + ) + }) + + it('rejects a timeout that exceeds the platform timer range', () => { + expect( + () => + new BASMRemote(ENDPOINT, TOPIC, async () => response({}), { + timeoutMs: 2147483648 + }) + ).toThrow(new TypeError('BASM timeout exceeds timer range')) + }) + + it('reads a response whose advertised length is within the configured cap', async () => { + const payload = JSON.stringify({ topic: TOPIC, blockHeight: -1, tac: ZERO }) + const remote = new BASMRemote( + ENDPOINT, + TOPIC, + async () => + new Response(payload, { + headers: { 'content-length': String(Buffer.byteLength(payload)) } + }), + { maxResponseBytes: Buffer.byteLength(payload) } + ) + + await expect(remote.requestTopicAnchorTip()).resolves.toEqual({ + topic: TOPIC, + blockHeight: -1, + tac: ZERO + }) + }) + + it('reads a body-less response through text() when headers are absent', async () => { + const payload = JSON.stringify({ topic: TOPIC, blockHeight: -1, tac: ZERO }) + const remote = new BASMRemote(ENDPOINT, TOPIC, async () => { + return { + ok: true, + status: 200, + body: null, + text: async () => payload + } as unknown as Response + }) + + await expect(remote.requestTopicAnchorTip()).resolves.toEqual({ + topic: TOPIC, + blockHeight: -1, + tac: ZERO + }) + }) + + it('rejects a body-less response whose decoded text exceeds the byte cap', async () => { + const remote = new BASMRemote( + ENDPOINT, + TOPIC, + async () => { + return { + ok: true, + status: 200, + headers: { get: () => null }, + body: undefined, + text: async () => '{"topic":"tm_example"}' + } as unknown as Response + }, + { maxResponseBytes: 4 } + ) + + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ + code: 'BASM_RESOURCE_LIMIT' + }) + }) + + it('cancels an oversized advertised body even when cancel rejects', async () => { + const remote = new BASMRemote( + ENDPOINT, + TOPIC, + async () => { + return { + ok: true, + status: 200, + headers: { get: () => '64' }, + body: { + cancel: async () => { + throw new Error('cancel failed') + } + }, + text: async () => '{"topic":"tm_example"}' + } as unknown as Response + }, + { maxResponseBytes: 8 } + ) + + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ + code: 'BASM_RESOURCE_LIMIT' + }) + }) + + it('skips body cancel when an oversized advertised response has no body', async () => { + const remote = new BASMRemote( + ENDPOINT, + TOPIC, + async () => { + return { + ok: true, + status: 200, + headers: { get: () => '64' }, + body: null, + text: async () => '{"topic":"tm_example"}' + } as unknown as Response + }, + { maxResponseBytes: 8 } + ) + + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ + code: 'BASM_RESOURCE_LIMIT' + }) + }) + + it('classifies malformed JSON on a failed HTTP response as an HTTP error', async () => { + const remote = new BASMRemote(ENDPOINT, TOPIC, async () => new Response('{', { status: 503 })) + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ + code: 'BASM_HTTP_ERROR' + }) + }) + + it.each([ + ['resolves', async () => {}], + [ + 'rejects', + async () => { + throw new Error('cancel failed') + } + ] + ])('cancels a late response body whose cancel %s after abort', async (_label, cancel) => { + let completeFetch: ((value: Response) => void) | undefined + let cancelled = false + const remote = new BASMRemote( + ENDPOINT, + TOPIC, + async () => + await new Promise(resolve => { + completeFetch = resolve + }), + { timeoutMs: 20 } + ) + + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ + code: 'BASM_TIMEOUT' + }) + + completeFetch?.({ + ok: true, + status: 200, + headers: { get: () => null }, + body: { + cancel: async () => { + cancelled = true + await cancel() + } + }, + text: async () => '' + } as unknown as Response) + + const deadline = Date.now() + 1000 + while (!cancelled && Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 10)) + } + expect(cancelled).toBe(true) + }) + + it('does not require a body when cancelling a late aborted response', async () => { + let completeFetch: ((value: Response) => void) | undefined + const remote = new BASMRemote( + ENDPOINT, + TOPIC, + async () => + await new Promise(resolve => { + completeFetch = resolve + }), + { timeoutMs: 20 } + ) + + await expect(remote.requestTopicAnchorTip()).rejects.toMatchObject({ + code: 'BASM_TIMEOUT' + }) + expect(completeFetch).toBeDefined() + + completeFetch?.({ + ok: true, + status: 200, + headers: { get: () => null }, + body: null, + text: async () => '' + } as unknown as Response) + + await new Promise(resolve => setTimeout(resolve, 20)) + }) }) From 94d2e7c30b3401f02d4e05ca66c114a29fe68e15 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 00:22:15 -0500 Subject: [PATCH 082/168] fix(sonar): extract overlay chain-tracker guard from Wallet discovery S3776: discoverOverlayCertificates was 17 after the forceRefresh services check. The missing-services path is now requireOverlayChainTracker. --- packages/wallet/wallet-toolbox/src/Wallet.ts | 24 +++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/wallet/wallet-toolbox/src/Wallet.ts b/packages/wallet/wallet-toolbox/src/Wallet.ts index cc953cca2..c6bcb8ce7 100644 --- a/packages/wallet/wallet-toolbox/src/Wallet.ts +++ b/packages/wallet/wallet-toolbox/src/Wallet.ts @@ -828,15 +828,7 @@ export class Wallet implements WalletInterface, ProtoWallet { this._overlayEvidenceExpiryTimer.unref?.() } - private async discoverOverlayCertificates( - query: unknown, - cacheKey: string, - forceRefresh: boolean, - now: number - ): Promise { - // Use the wallet's existing network/chain configuration, never the overlay host's verdict. - // Wallets constructed without services have no chain tracker. Overlay lookup then yields - // no identities, except forceRefresh which still requires services (contact bypass). + private async requireOverlayChainTracker(forceRefresh: boolean) { if (this.services == null) { if (forceRefresh) { throw new WERR_INVALID_PARAMETER( @@ -844,9 +836,19 @@ export class Wallet implements WalletInterface, ProtoWallet { 'valid in constructor arguments to be retreived here.' ) } - return [] + return undefined } - const chainTracker = await this.services.getChainTracker() + return await this.services.getChainTracker() + } + + private async discoverOverlayCertificates( + query: unknown, + cacheKey: string, + forceRefresh: boolean, + now: number + ): Promise { + const chainTracker = await this.requireOverlayChainTracker(forceRefresh) + if (chainTracker == null) return [] if (this._identityEvidenceClosed) return [] const chainNamespace = `wallet:${this.chain}` if ( From d8b8df54ee376815fb885e3c93200ad95bb640bf Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 00:26:48 -0500 Subject: [PATCH 083/168] test(sdk): cover overlay discovery helpers for patch coverage --- .../__tests/LookupDiscovery.test.ts | 146 +++++++++++++ .../__tests/LookupHostQueue.test.ts | 62 ++++++ .../__tests/LookupResolver.dynamic.test.ts | 195 +++++++++++++++++- .../__tests/LookupResources.test.ts | 64 ++++++ .../__tests/LookupResponseReader.test.ts | 128 ++++++++++++ 5 files changed, 587 insertions(+), 8 deletions(-) create mode 100644 packages/sdk/src/overlay-tools/__tests/LookupDiscovery.test.ts create mode 100644 packages/sdk/src/overlay-tools/__tests/LookupHostQueue.test.ts create mode 100644 packages/sdk/src/overlay-tools/__tests/LookupResources.test.ts diff --git a/packages/sdk/src/overlay-tools/__tests/LookupDiscovery.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupDiscovery.test.ts new file mode 100644 index 000000000..c63acdc52 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupDiscovery.test.ts @@ -0,0 +1,146 @@ +import { LookupDiscovery } from '../LookupDiscovery.js' +import { LookupResourceLimitError, lookupLimits } from '../LookupResources.js' + +const limits = lookupLimits({ + maxHosts: 4, + maxHostsPerTracker: 2, + maxTrackers: 4, + hostConcurrency: 2, + trackerConcurrency: 1, + maxResponseBytes: 100, + maxTotalBytes: 20, + maxOutputs: 8, + maxEvidenceOutputs: 8, + maxEvidenceBytes: 100 +}) + +function deferred(): { + promise: Promise + resolve: (value: T) => void + reject: (reason?: unknown) => void +} { + let resolve!: (value: T) => void + let reject!: (reason?: unknown) => void + const promise = new Promise((res, rej) => { + resolve = res + reject = rej + }) + return { promise, resolve, reject } +} + +describe('LookupDiscovery', () => { + it('rejects further byte charges after the last subscriber abandons discovery', async () => { + const started = deferred<(bytes: number) => void>() + const hang = deferred() + const discovery = new LookupDiscovery( + ['https://tracker.example'], + limits, + async (_tracker, _signal, consume) => { + started.resolve(consume) + return await hang.promise + }, + () => {} + ) + const unsubscribe = discovery.subscribe(() => {}) + const consume = await started.promise + unsubscribe() + + expect(() => consume(1)).toThrow('Lookup resource limit reached: abandoned') + + hang.resolve([]) + }) + + it('charges tracker bytes until maxTotalBytes then records the limit without a tracker failure', async () => { + const discovery = new LookupDiscovery( + ['https://tracker.example'], + limits, + async (_tracker, _signal, consume) => { + consume(10) + consume(11) + return [] + }, + () => {} + ) + const finished = deferred() + discovery.subscribe(state => { + if (state.done) finished.resolve() + }) + await finished.promise + + expect(discovery.state.receivedBytes).toBe(10) + expect(discovery.state.limitsHit.has('maxTotalBytes')).toBe(true) + expect(discovery.state.trackersFailed).toBe(0) + expect(discovery.state.trackersCompleted).toBe(1) + }) + + it('skips invalid, duplicate, and over-share hosts from one tracker', async () => { + const discovery = new LookupDiscovery( + ['https://tracker.example'], + limits, + async () => [ + 'not-a-url', + 'ftp://blocked.example', + 'https://user:pass@secret.example', + 'https://host.example', + 'https://host.example/', + 'https://second.example', + 'https://third.example' + ], + () => {} + ) + const finished = deferred() + discovery.subscribe(state => { + if (state.done) finished.resolve() + }) + await finished.promise + + expect(discovery.state.sources.get('https://tracker.example')).toEqual([ + 'https://host.example', + 'https://second.example' + ]) + expect(discovery.state.skippedHosts).toBeGreaterThanOrEqual(3) + expect(discovery.state.limitsHit.has('maxHostsPerTracker')).toBe(true) + }) + + it('records a resource-limit error from lookup without counting a tracker failure', async () => { + const discovery = new LookupDiscovery( + ['https://tracker.example'], + limits, + async () => { + throw new LookupResourceLimitError('maxResponseBytes') + }, + () => {} + ) + const finished = deferred() + discovery.subscribe(state => { + if (state.done) finished.resolve() + }) + await finished.promise + + expect(discovery.state.limitsHit.has('maxResponseBytes')).toBe(true) + expect(discovery.state.trackersFailed).toBe(0) + }) + + it('does not record hosts after the last subscriber abandons an in-flight tracker', async () => { + const started = deferred() + const release = deferred() + const discovery = new LookupDiscovery( + ['https://tracker.example'], + limits, + async () => { + started.resolve() + return await release.promise + }, + () => {} + ) + const unsubscribe = discovery.subscribe(() => {}) + await started.promise + unsubscribe() + release.resolve(['https://late-host.example']) + while (discovery.state.trackersCompleted === 0) { + await new Promise(resolve => setImmediate(resolve)) + } + + expect(discovery.state.sources.size).toBe(0) + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupHostQueue.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupHostQueue.test.ts new file mode 100644 index 000000000..321b275fa --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupHostQueue.test.ts @@ -0,0 +1,62 @@ +import { LookupHostQueue } from '../LookupHostQueue.js' + +describe('LookupHostQueue', () => { + it('does not enqueue hosts after sources close or the queue is cancelled', async () => { + const ran: string[] = [] + const closed = new LookupHostQueue( + 8, + 1, + async host => { + ran.push(host) + }, + () => {} + ) + closed.finishSources() + closed.add('late', ['https://late.example']) + await closed.done + + const cancelled = new LookupHostQueue( + 8, + 1, + async host => { + ran.push(host) + }, + () => {} + ) + cancelled.cancel() + cancelled.add('late', ['https://cancelled.example']) + await cancelled.done + + expect(ran).toEqual([]) + }) + + it('ignores duplicate hosts and reports overflow past maxHosts', async () => { + const ran: string[] = [] + const skipped: Array<[number, boolean]> = [] + const queue = new LookupHostQueue( + 2, + 2, + async host => { + ran.push(host) + }, + (count, limited) => { + skipped.push([count, limited]) + } + ) + queue.add('tracker-a', [ + 'https://a.example', + 'https://a.example', + 'https://b.example', + 'https://c.example' + ]) + queue.add('tracker-b', ['https://a.example', 'https://d.example']) + queue.finishSources() + await queue.done + + expect(ran).toEqual(['https://a.example', 'https://b.example']) + expect(skipped).toEqual([ + [1, true], + [1, true] + ]) + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts index ac2959eb6..ed59565af 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts @@ -1,4 +1,7 @@ -import LookupResolver, { LookupAnswerProgress } from '../LookupResolver' +import LookupResolver, { + HTTPSOverlayLookupFacilitator, + LookupAnswerProgress +} from '../LookupResolver' import { getOverlayHostReputationTracker } from '../HostReputationTracker' import OverlayAdminTokenTemplate from '../OverlayAdminTokenTemplate' import { CompletedProtoWallet } from '../../auth/certificates/__tests/CompletedProtoWallet' @@ -859,7 +862,9 @@ describe('LookupResolver dynamic discovery', () => { }) const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) - const tight = resolver.query({ service, query: { n: 1 } }, undefined, { limits: { maxHosts: 1 } }) + const tight = resolver.query({ service, query: { n: 1 } }, undefined, { + limits: { maxHosts: 1 } + }) await jest.runAllTimersAsync() await tight @@ -901,14 +906,18 @@ describe('LookupResolver dynamic discovery', () => { return { type: 'output-list' as const, outputs: [] } }) const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) - const first = resolver.query$({ service, query: { n: 1 } }, undefined, { - limits: { maxHosts: 1 } - })[Symbol.asyncIterator]() + const first = resolver + .query$({ service, query: { n: 1 } }, undefined, { + limits: { maxHosts: 1 } + }) + [Symbol.asyncIterator]() const firstPending = first.next() await Promise.resolve() - const second = resolver.query$({ service, query: { n: 2 } }, undefined, { - limits: { maxHosts: 2 } - })[Symbol.asyncIterator]() + const second = resolver + .query$({ service, query: { n: 2 } }, undefined, { + limits: { maxHosts: 2 } + }) + [Symbol.asyncIterator]() const secondPending = second.next() await Promise.resolve() @@ -982,4 +991,174 @@ describe('LookupResolver dynamic discovery', () => { outputs: [] }) }) + + it('reuses a covering cache that omitted optional discovery metadata', async () => { + const host = 'https://cached-meta.example' + const tracker = 'https://cached-meta-tracker.example' + const service = 'ls_cached_meta' + const beef = makeBeef(201) + const lookup = jest.fn(async (url: string) => { + if (url === tracker) throw new Error('tracker should not run') + return { type: 'output-list' as const, outputs: [{ beef, outputIndex: 0 }] } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + const limits = (resolver as any).limits + ;(resolver as any).hostsCache.set(service, { + maxHosts: limits.maxHosts, + maxHostsPerTracker: limits.maxHostsPerTracker, + maxTrackers: limits.maxTrackers, + maxResponseBytes: limits.maxResponseBytes, + maxTotalBytes: limits.maxTotalBytes, + maxOutputs: limits.maxOutputs, + hosts: [host], + expiresAt: Date.now() + 60_000 + }) + const pending = resolver.query({ service, query: {} }) + await jest.runAllTimersAsync() + await expect(pending).resolves.toEqual({ + type: 'output-list', + outputs: [{ beef, outputIndex: 0 }] + }) + expect(lookup.mock.calls.map(([url]) => url)).toEqual([host]) + }) + + it('refreshes when a planted cache is missing any discovery bound', async () => { + const tracker = 'https://missing-bound-tracker.example' + const host = 'https://missing-bound-host.example' + const receipt = await slapReceipt(210, host, 'ls_missing_bound') + const lookup = jest.fn(async (url: string) => { + if (url === tracker) return { type: 'output-list' as const, outputs: [receipt] } + return { type: 'output-list' as const, outputs: [] } + }) + const resolver = new LookupResolver({ facilitator: { lookup }, slapTrackers: [tracker] }) + const limits = (resolver as any).limits + const missingFields = [ + 'maxHosts', + 'maxHostsPerTracker', + 'maxTrackers', + 'maxResponseBytes', + 'maxTotalBytes', + 'maxOutputs' + ] as const + for (const missing of missingFields) { + const service = `ls_missing_${missing}` + const cached: Record = { + maxHosts: limits.maxHosts, + maxHostsPerTracker: limits.maxHostsPerTracker, + maxTrackers: limits.maxTrackers, + maxResponseBytes: limits.maxResponseBytes, + maxTotalBytes: limits.maxTotalBytes, + maxOutputs: limits.maxOutputs, + hosts: [host], + expiresAt: Date.now() + 60_000 + } + delete cached[missing] + ;(resolver as any).hostsCache.set(service, cached) + const pending = resolver.query({ service, query: {} }) + await jest.runAllTimersAsync() + await pending + } + expect(lookup.mock.calls.filter(([url]) => url === tracker).length).toBe(missingFields.length) + }) + + it('rejects a deadline outside the accepted range', async () => { + const resolver = new LookupResolver({ + facilitator: { lookup: async () => ({ type: 'output-list' as const, outputs: [] }) }, + hostOverrides: { ls_deadline_range: ['https://deadline-range.example'] } + }) + await expect( + resolver.query({ service: 'ls_deadline_range', query: {} }, undefined, { deadlineMs: -1 }) + ).rejects.toBeInstanceOf(RangeError) + await expect( + resolver.query({ service: 'ls_deadline_range', query: {} }, undefined, { + deadlineMs: 2_147_483_648 + }) + ).rejects.toBeInstanceOf(RangeError) + await expect( + resolver.query({ service: 'ls_deadline_range', query: {} }, undefined, { + deadlineMs: Number.NaN + }) + ).rejects.toBeInstanceOf(RangeError) + }) + + it('cancels immediately when the caller signal is already aborted', async () => { + const lookup = jest.fn(async () => ({ type: 'output-list' as const, outputs: [] })) + const resolver = new LookupResolver({ + facilitator: { lookup }, + hostOverrides: { ls_preabort: ['https://preabort.example'] } + }) + const controller = new AbortController() + controller.abort() + const received: LookupAnswerProgress[] = [] + const pending = (async () => { + for await (const item of resolver.query$({ service: 'ls_preabort', query: {} }, undefined, { + signal: controller.signal + })) { + received.push(item) + } + })() + await jest.runAllTimersAsync() + await pending + expect(lookup).not.toHaveBeenCalled() + expect(received.at(-1)).toMatchObject({ isFinal: true, terminalReason: 'cancelled' }) + }) + + it('aborts an in-flight query$ when the iterator throws', async () => { + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_iter_throw: ['https://iter-throw.example'] } + }) + const iterator = resolver + .query$({ service: 'ls_iter_throw', query: {} }) + [Symbol.asyncIterator]() + const first = iterator.next() + await Promise.resolve() + await expect(iterator.throw(new Error('iterator failed'))).rejects.toThrow('iterator failed') + await first.catch(() => undefined) + }) + + it('shallow-copies a question that structuredClone cannot clone for a custom facilitator', async () => { + const query: { nested: { n: number }; fn?: () => number } = { nested: { n: 1 }, fn: () => 1 } + const lookup = jest.fn(async (_url: string, question: { query: typeof query }) => { + expect(question.query).toEqual(query) + expect(question.query).not.toBe(query) + expect(question.query.nested).toBe(query.nested) + return { type: 'output-list' as const, outputs: [] } + }) + const resolver = new LookupResolver({ + facilitator: { lookup }, + hostOverrides: { ls_clone: ['https://clone.example'] } + }) + const pending = resolver.query({ service: 'ls_clone', query }) + await jest.runAllTimersAsync() + await pending + expect(lookup).toHaveBeenCalledTimes(1) + }) + + it('re-serializes a non-cloneable question for the HTTPS facilitator', async () => { + const fetchClient = jest.fn( + async () => + new Response(JSON.stringify({ type: 'output-list', outputs: [] }), { + status: 200, + headers: { 'content-type': 'application/json' } + }) + ) + const resolver = new LookupResolver({ + facilitator: new HTTPSOverlayLookupFacilitator(fetchClient as any, true), + hostOverrides: { ls_https_clone: ['https://https-clone.example'] } + }) + const pending = resolver.query({ + service: 'ls_https_clone', + query: { fn: () => 1 } + } as any) + await jest.runAllTimersAsync() + await pending + expect(fetchClient).toHaveBeenCalled() + }) }) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResources.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResources.test.ts new file mode 100644 index 000000000..4611a83d3 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResources.test.ts @@ -0,0 +1,64 @@ +import { + DEFAULT_LOOKUP_LIMITS, + lookupAbortError, + lookupLimits, + normalizeLookupHost, + withLookupAbort +} from '../LookupResources.js' + +describe('lookupLimits', () => { + it('accepts positive safe integers and rejects every other shape', () => { + expect(lookupLimits(undefined).maxHosts).toBe(DEFAULT_LOOKUP_LIMITS.maxHosts) + expect(lookupLimits({ maxHosts: 3 }).maxHosts).toBe(3) + expect(() => lookupLimits({ maxHosts: 0 })).toThrow(RangeError) + expect(() => lookupLimits({ maxHosts: -1 })).toThrow( + /Lookup limit maxHosts must be a positive safe integer/ + ) + expect(() => lookupLimits({ hostConcurrency: 1.5 })).toThrow(RangeError) + expect(() => lookupLimits({ maxTrackers: Number.NaN })).toThrow(RangeError) + expect(() => lookupLimits({ maxOutputs: Infinity })).toThrow(RangeError) + expect(() => lookupLimits({ maxTotalBytes: Number.MAX_SAFE_INTEGER + 1 })).toThrow(RangeError) + expect(() => lookupLimits({ maxEvidenceBytes: '8' as unknown as number })).toThrow(RangeError) + }) +}) + +describe('normalizeLookupHost', () => { + it('rejects non-strings, overlong values, credentials, and non-http URLs', () => { + expect(normalizeLookupHost(undefined as unknown as string)).toBeNull() + expect(normalizeLookupHost(`https://example.com/${'a'.repeat(2048)}`)).toBeNull() + expect(normalizeLookupHost('ftp://example.com')).toBeNull() + expect(normalizeLookupHost('https://user@example.com')).toBeNull() + expect(normalizeLookupHost('https://user:pass@example.com')).toBeNull() + expect(normalizeLookupHost('https://:secret@example.com')).toBeNull() + }) + + it('drops query and fragment unless parameters are explicitly allowed', () => { + expect(normalizeLookupHost('https://example.com/?q=1')).toBeNull() + expect(normalizeLookupHost('https://example.com/#frag')).toBeNull() + expect(normalizeLookupHost('https://example.com/?q=1', true)).toBe('https://example.com/?q=1') + expect(normalizeLookupHost('https://example.com/#frag', true)).toBe('https://example.com/#frag') + }) + + it('returns null for values that are not parseable as URLs', () => { + expect(normalizeLookupHost('not a url')).toBeNull() + expect(normalizeLookupHost('https://[')).toBeNull() + expect(normalizeLookupHost('')).toBeNull() + }) +}) + +describe('withLookupAbort', () => { + it('drops an already-aborted waiter and does not leak a rejecting transport', async () => { + const controller = new AbortController() + controller.abort() + const rejected = Promise.reject(new Error('late transport')) + + await expect(withLookupAbort(Promise.resolve('ok'), controller.signal)).rejects.toMatchObject({ + name: 'AbortError', + message: 'Lookup cancelled' + }) + await expect(withLookupAbort(rejected, controller.signal)).rejects.toMatchObject({ + name: 'AbortError' + }) + expect(lookupAbortError().name).toBe('AbortError') + }) +}) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResponseReader.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResponseReader.test.ts index 2cdf8d075..3608a64ac 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResponseReader.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResponseReader.test.ts @@ -240,4 +240,132 @@ describe('readLookupResponseBytes', () => { readLookupResponseBytes(responseForReader(reader), { maxResponseBytes: 10 }) ).rejects.toBe(readFailure) }) + + it('rejects a maxResponseBytes that is not a non-negative safe integer', async () => { + const response = responseForReader(readerForChunks([])) + await expect( + readLookupResponseBytes(response, { maxResponseBytes: -1 }) + ).rejects.toBeInstanceOf(RangeError) + await expect(readLookupResponseBytes(response, { maxResponseBytes: 1.5 })).rejects.toThrow( + 'maxResponseBytes must be a non-negative safe integer' + ) + await expect( + readLookupResponseBytes(response, { maxResponseBytes: Number.MAX_SAFE_INTEGER + 1 }) + ).rejects.toBeInstanceOf(RangeError) + }) + + it('treats a malformed Content-Length as unknown and still reads the body', async () => { + const bytes = await readLookupResponseBytes( + responseForReader(readerForChunks([new Uint8Array([9])]), '1e6'), + { maxResponseBytes: 1 } + ) + expect(bytes).toEqual(new Uint8Array([9])) + }) + + it('returns an empty body when the response has no stream', async () => { + const headers = new Headers() + const empty = { body: null, headers } as unknown as Response + await expect(readLookupResponseBytes(empty, { maxResponseBytes: 0 })).resolves.toEqual( + new Uint8Array(0) + ) + + headers.set('content-length', '4') + await expect( + readLookupResponseBytes({ body: null, headers } as unknown as Response, { + maxResponseBytes: 1 + }) + ).rejects.toMatchObject({ name: 'LookupResourceLimitError', limit: 'maxResponseBytes' }) + }) + + it('rejects an already-aborted empty body using the AbortError fallback when no reason is set', async () => { + const signal = { + aborted: true, + reason: undefined, + addEventListener: () => undefined, + removeEventListener: () => undefined + } as unknown as AbortSignal + await expect( + readLookupResponseBytes({ body: null, headers: new Headers() } as unknown as Response, { + maxResponseBytes: 0, + signal + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) + + it('rejects an already-aborted stream before the first read', async () => { + const controller = new AbortController() + const read = jest.fn() + controller.abort(new Error('already aborted')) + const reader = { + read, + cancel: async () => undefined, + releaseLock: () => undefined + } as unknown as ReadableStreamDefaultReader + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 10, + signal: controller.signal + }) + ).rejects.toThrow('already aborted') + expect(read).not.toHaveBeenCalled() + }) + + it('treats a missing chunk value as empty input', async () => { + const consumed: number[] = [] + const reader = { + read: jest + .fn() + .mockResolvedValueOnce({ done: false, value: undefined }) + .mockResolvedValueOnce({ done: true, value: undefined }), + cancel: async () => undefined, + releaseLock: () => undefined + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 4, + consumeBytes: byteCount => consumed.push(byteCount) + }) + ).resolves.toEqual(new Uint8Array(0)) + expect(consumed).toEqual([]) + }) + + it('rejects a later read when consumeBytes aborts the signal', async () => { + const controller = new AbortController() + const cancel = jest.fn(async () => undefined) + const reader = { + read: jest + .fn() + .mockResolvedValueOnce({ done: false, value: new Uint8Array([1]) }) + .mockResolvedValueOnce({ done: false, value: new Uint8Array([2]) }), + cancel, + releaseLock: jest.fn() + } as unknown as ReadableStreamDefaultReader + + await expect( + readLookupResponseBytes(responseForReader(reader), { + maxResponseBytes: 10, + signal: controller.signal, + consumeBytes: () => controller.abort() + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(cancel).toHaveBeenCalledTimes(1) + }) + + it('rejects an abort that races listener registration', async () => { + const signal = { + aborted: false, + reason: new Error('raced abort'), + addEventListener: () => { + signal.aborted = true + }, + removeEventListener: () => undefined + } + await expect( + readLookupResponseBytes(responseForReader(readerForChunks([new Uint8Array([1])])), { + maxResponseBytes: 10, + signal: signal as unknown as AbortSignal + }) + ).rejects.toThrow('raced abort') + }) }) From 0a847b56c7e51b6aa2e9be8b66fbaf84268454c7 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 00:33:31 -0500 Subject: [PATCH 084/168] test(overlay): cover mongo storage helpers for patch coverage --- docs/packages/overlays/overlay.md | 15 +- packages/overlays/overlay/CHANGELOG.md | 1 + packages/overlays/overlay/README.md | 11 +- packages/overlays/overlay/mod.ts | 1 + packages/overlays/overlay/package.json | 2 +- .../src/__tests/StorageContract.test.ts | 6 +- .../__tests/mongo/MongoGuardBranches.test.ts | 561 ++++++++++++++++++ .../__tests/mongo/MongoPayloadStore.test.ts | 162 +++++ .../src/__tests/mongo/MongoReadGuards.test.ts | 30 + .../src/__tests/mongo/MongoSchema.test.ts | 36 ++ .../mongo/MongoTransactionRunner.test.ts | 26 + .../overlays/overlay/src/storage/Storage.ts | 5 + 12 files changed, 843 insertions(+), 13 deletions(-) create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoGuardBranches.test.ts diff --git a/docs/packages/overlays/overlay.md b/docs/packages/overlays/overlay.md index 38a99b04a..e6195bc2d 100644 --- a/docs/packages/overlays/overlay.md +++ b/docs/packages/overlays/overlay.md @@ -5,8 +5,8 @@ kind: package domain: overlays npm: '@bsv/overlay' version: '2.4.0' -last_updated: '2026-09-08' -last_verified: '2026-09-08' +last_updated: '2026-09-09' +last_verified: '2026-09-09' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay' status: stable @@ -79,10 +79,13 @@ const result = await engine.lookup({ ## Optional persistence contracts The package exports an additive `AdmissionStorage` capability, semantic identity -helpers and recovery fence/cursor predicates. These define the local durable -receipt and pending index/propagation boundary for future adapters. Current -`Engine.submit`, its early STEAK callback and Knex storage do not use the -capability. See the [persistence v1 specification](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/persistence-v1.md) +helpers and recovery fence/cursor predicates. `storageHasAdmission` reports +whether the optional `Storage.admission` field is present; `getAdmissionStorage` +additionally requires the v1 protocol and both commit and reconciliation +methods. These define the local durable receipt and pending index/propagation +boundary for future adapters. Current `Engine.submit`, its early STEAK callback +and Knex storage do not use the capability. See the [persistence v1 +specification](https://github.com/bsv-blockchain/ts-stack/blob/main/specs/overlay/persistence-v1.md) for the shared fixtures and explicit limits. No consumer migration is required. ## Optional Mongo foundation diff --git a/packages/overlays/overlay/CHANGELOG.md b/packages/overlays/overlay/CHANGELOG.md index 64c003ec8..8660374bb 100644 --- a/packages/overlays/overlay/CHANGELOG.md +++ b/packages/overlays/overlay/CHANGELOG.md @@ -21,6 +21,7 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] - Adds optional atomic admission/receipt and recovery contracts with portable identity, exact integer and fencing fixtures. Existing Engine/Knex paths are unchanged; no migration is required. +- Adds `storageHasAdmission` for the optional `Storage.admission` field. - Adds an opt-in MongoDB schema, payload-publication, reference-guard, and payload-GC foundation behind an optional `mongodb` peer. It does not activate MongoDB in Engine, select a new default adapter, or provide an admission diff --git a/packages/overlays/overlay/README.md b/packages/overlays/overlay/README.md index 891b16f70..734052799 100644 --- a/packages/overlays/overlay/README.md +++ b/packages/overlays/overlay/README.md @@ -82,11 +82,12 @@ applications should prefer the root entry point wherever possible. ## Optional persistence capability `AdmissionStorage` defines an additive v1 atomic admission contract for future -adapters. `getAdmissionStorage(storage)` detects an explicit provider with both -commit and reconciliation methods. Existing Knex and injected legacy adapters -remain supported; their individual methods do not imply atomic submission. -Current `Engine.submit` does not call this capability, and its early STEAK -callback is not a durable commit receipt. +adapters. `storageHasAdmission(storage)` reports whether the optional +`admission` field is present. `getAdmissionStorage(storage)` additionally +requires the v1 protocol and both commit and reconciliation methods. Existing +Knex and injected legacy adapters remain supported; their individual methods do +not imply atomic submission. Current `Engine.submit` does not call this +capability, and its early STEAK callback is not a durable commit receipt. The contract separates local commit, index visibility and propagation. It binds operation identity to verified transaction, topic/policy and off-chain context; diff --git a/packages/overlays/overlay/mod.ts b/packages/overlays/overlay/mod.ts index 7afaf8b61..88d29fb33 100644 --- a/packages/overlays/overlay/mod.ts +++ b/packages/overlays/overlay/mod.ts @@ -6,6 +6,7 @@ export type { TopicAdmittanceContext, TopicManager } from './src/TopicManager.js // Interfaces and structures export type { Storage, AppliedTransaction } from './src/storage/Storage.js' +export { storageHasAdmission } from './src/storage/Storage.js' export * from './src/storage/AdmissionStorage.js' export * from './src/storage/RecoveryContract.js' export type { Output } from './src/Output.js' diff --git a/packages/overlays/overlay/package.json b/packages/overlays/overlay/package.json index cdad98132..406b7e56e 100644 --- a/packages/overlays/overlay/package.json +++ b/packages/overlays/overlay/package.json @@ -81,7 +81,7 @@ "format:check": "pnpm --workspace-root exec prettier --check \"packages/overlays/overlay/{README.md,jest.config.js,mod.ts,package.json,tsconfig*.json}\"", "lint": "oxlint mod.ts src jest.config.js --deny-warnings", "lint:ci": "pnpm lint", - "pack:check": "pnpm build && node ../../../scripts/check-package-artifact.mjs . --exports admissionSemanticDigest,canAdvanceGaspCursor,getAdmissionStorage,isRecoveryLeaseCurrent,isReplaySafeProjection,parseStorageUint64,parseStorageOutputIndex,BASM_ZERO_HASH,Engine,KnexStorage,KnexStorageMigrations,computeBasmRoot,computeTac,extractMerkleProofMetadata,serializeErrorForLog,serializeLogValue --entry-exports \"./storage=;./storage/knex/KnexStorage=KnexStorage\"", + "pack:check": "pnpm build && node ../../../scripts/check-package-artifact.mjs . --exports admissionSemanticDigest,canAdvanceGaspCursor,getAdmissionStorage,isRecoveryLeaseCurrent,isReplaySafeProjection,parseStorageUint64,parseStorageOutputIndex,storageHasAdmission,BASM_ZERO_HASH,Engine,KnexStorage,KnexStorageMigrations,computeBasmRoot,computeTac,extractMerkleProofMetadata,serializeErrorForLog,serializeLogValue --entry-exports \"./storage=storageHasAdmission;./storage/knex/KnexStorage=KnexStorage\"", "typecheck": "tsc --project tsconfig.base.json --noEmit --incremental false", "build": "tsc -b && tsconfig-to-dual-package tsconfig.cjs.json", "dev": "tsc -b -w", diff --git a/packages/overlays/overlay/src/__tests/StorageContract.test.ts b/packages/overlays/overlay/src/__tests/StorageContract.test.ts index a37ae19c3..e613d7315 100644 --- a/packages/overlays/overlay/src/__tests/StorageContract.test.ts +++ b/packages/overlays/overlay/src/__tests/StorageContract.test.ts @@ -10,6 +10,7 @@ import { parseStorageUint64, type AdmissionIdentity } from '../storage/AdmissionStorage.js' +import { storageHasAdmission, type Storage } from '../storage/Storage.js' import { canAdvanceGaspCursor, isRecoveryLeaseCurrent, @@ -121,7 +122,9 @@ describe('S01 portable persistence contract', () => { test('legacy Knex CRUD cannot advertise atomic admission', () => { const knex = jest.fn() as unknown as Knex - expect(getAdmissionStorage(new KnexStorage(knex))).toBeUndefined() + const storage = new KnexStorage(knex) + expect(getAdmissionStorage(storage)).toBeUndefined() + expect(storageHasAdmission(storage)).toBe(false) expect(knex).not.toHaveBeenCalled() }) @@ -145,6 +148,7 @@ describe('S01 portable persistence contract', () => { expect(getAdmissionStorage(storage)).toBeUndefined() } expect(getAdmissionStorage({ admission })).toBe(admission) + expect(storageHasAdmission({ admission } as Storage)).toBe(true) expect(admission.commitAdmission).not.toHaveBeenCalled() }) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoGuardBranches.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoGuardBranches.test.ts new file mode 100644 index 000000000..6b1a47664 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoGuardBranches.test.ts @@ -0,0 +1,561 @@ +import { randomUUID } from 'node:crypto' +import type { ClientSession, Db } from 'mongodb' +import { + admissionSemanticDigest, + type AdmissionIdentity, + type StorageScope +} from '../../storage/AdmissionStorage.js' +import { + MongoPayloadStore, + type MongoPayloadStoreOptions +} from '../../storage/mongo/MongoPayloadStore.js' +import { + MongoReadGuardConflictError, + MongoReadGuards +} from '../../storage/mongo/MongoReadGuards.js' +import { + bootstrapMongoOverlay, + mongoChainKey, + mongoNodeKey, + mongoRecordKey, + type MongoChainScope +} from '../../storage/mongo/MongoSchema.js' +import { + MongoTransactionRunner, + type MongoTransactionRequest +} from '../../storage/mongo/MongoTransactionRunner.js' + +const scope: StorageScope = { + network: 'testnet', + genesisHash: '11'.repeat(32), + nodeId: 's02-node-a' +} + +const dummyDb = { collection: () => ({}) } as unknown as Db + +const fakeSession = (inTransaction = true): ClientSession => + ({ inTransaction: () => inTransaction }) as ClientSession + +const digest = 'aa'.repeat(32) + +function payloadStore(db: Db = dummyDb, options?: MongoPayloadStoreOptions) { + return new MongoPayloadStore(db, scope, options) +} + +function identity(): AdmissionIdentity { + return { + scope, + txid: '22'.repeat(32), + mode: 'live', + contextDigest: '33'.repeat(32), + topics: [{ topic: 'topic.a', policyId: 'v1' }] + } +} + +function request(): MongoTransactionRequest { + const value = identity() + const semanticDigest = admissionSemanticDigest(value) + const operationId = randomUUID() + return { + identity: value, + key: { scope, operationId, semanticDigest }, + receipt: { + operationId, + semanticDigest, + durability: 'atomic-local', + steak: '{"topic.a":{"outputsToAdmit":[0]}}', + indexes: [{ target: 'lookup', state: 'pending' }], + propagation: 'pending' + } + } +} + +describe('Mongo schema key and topology guards', () => { + test('rejects empty, oversized, and too-many-part record keys', () => { + expect(() => mongoRecordKey()).toThrow('Invalid Mongo record key') + expect(() => mongoRecordKey(...Array.from({ length: 33 }, (_, index) => `p${index}`))).toThrow( + 'Invalid Mongo record key' + ) + expect(() => + mongoRecordKey('a'.repeat(1024), 'b'.repeat(1024), 'c'.repeat(1024), 'd'.repeat(1024)) + ).toThrow('Mongo record key is too large') + }) + + test('rejects invalid chain and node scopes', () => { + expect(() => mongoChainKey(null as unknown as MongoChainScope)).toThrow( + 'Invalid Mongo chain scope' + ) + expect(() => mongoChainKey({ network: 'test', genesisHash: 'zz'.repeat(32) })).toThrow( + 'Invalid Mongo chain scope' + ) + expect(() => mongoChainKey({ network: 'n'.repeat(129), genesisHash: 'aa'.repeat(32) })).toThrow( + 'Invalid Mongo chain scope' + ) + expect(() => mongoChainKey({ network: '\ud800', genesisHash: 'aa'.repeat(32) })).toThrow( + 'Invalid Mongo record key component' + ) + expect(() => mongoNodeKey({ ...scope, nodeId: '' })).toThrow( + 'Invalid Mongo record key component' + ) + expect(() => mongoNodeKey({ ...scope, nodeId: 'x\u0000y' })).toThrow( + 'Invalid Mongo record key component' + ) + expect(() => mongoNodeKey({ ...scope, nodeId: 'n'.repeat(1025) })).toThrow( + 'Invalid Mongo record key component' + ) + }) + + test('requires an unsharded writable replica-set primary before schema work', async () => { + const command = jest.fn() + const db = { command } as unknown as Db + command.mockResolvedValueOnce({ msg: 'isdbgrid', setName: 'rs0', isWritablePrimary: true }) + await expect(bootstrapMongoOverlay(db, scope)).rejects.toThrow('unsharded replica set') + command.mockResolvedValueOnce({ setName: '', isWritablePrimary: true }) + await expect(bootstrapMongoOverlay(db, scope)).rejects.toThrow('unsharded replica set') + command.mockResolvedValueOnce({ setName: 'rs0', isWritablePrimary: false }) + await expect(bootstrapMongoOverlay(db, scope)).rejects.toThrow('writable primary') + command.mockResolvedValueOnce({ isWritablePrimary: true }) + await expect(bootstrapMongoOverlay(db, scope)).rejects.toThrow('unsharded replica set') + }) +}) + +describe('Mongo payload store input and operation guards', () => { + test('rejects constructor bounds that cannot be represented safely', () => { + expect(() => payloadStore(dummyDb, { inlineCeilingBytes: 0 })).toThrow('inline ceiling') + expect(() => payloadStore(dummyDb, { inlineCeilingBytes: 1024 * 1024 + 1 })).toThrow( + 'inline ceiling' + ) + expect(() => payloadStore(dummyDb, { inlineCeilingBytes: 1.5 })).toThrow('inline ceiling') + expect(() => payloadStore(dummyDb, { gridFsChunkBytes: 0 })).toThrow('GridFS chunk size') + expect(() => payloadStore(dummyDb, { gridFsChunkBytes: 255 * 1024 + 1 })).toThrow( + 'GridFS chunk size' + ) + expect(() => payloadStore(dummyDb, { uploadLeaseMs: 0 })).toThrow('upload lease') + expect(() => payloadStore(dummyDb, { maxPayloadBytes: BigInt(0) })).toThrow( + 'payload byte bound' + ) + expect(() => payloadStore(dummyDb, { maxPayloadBytes: BigInt(1) << BigInt(64) })).toThrow( + 'payload byte bound' + ) + }) + + test('rejects malformed publish inputs before touching storage', async () => { + const store = payloadStore() + const bytes = async function* (): AsyncIterable { + yield new Uint8Array([1]) + } + await expect( + store.publish({ kind: 'outbox-data', digest: 'zz', byteLength: '1', bytes: bytes() }) + ).rejects.toThrow('Invalid Mongo payload digest') + await expect( + store.publish({ kind: 'outbox-data', digest, byteLength: '01', bytes: bytes() }) + ).rejects.toThrow('Invalid Mongo payload byte length') + await expect( + store.publish({ + kind: 'outbox-data', + digest, + byteLength: '2', + bytes: bytes(), + txid: 'bb'.repeat(32) + }) + ).rejects.toThrow('Invalid Mongo raw transaction id') + await expect( + store.publish({ + kind: 'raw-transaction', + digest, + byteLength: '1', + bytes: bytes(), + txid: 'zz' + }) + ).rejects.toThrow('Invalid Mongo raw transaction id') + const bounded = payloadStore(dummyDb, { maxPayloadBytes: BigInt(1) }) + await expect( + bounded.publish({ kind: 'outbox-data', digest, byteLength: '2', bytes: bytes() }) + ).rejects.toThrow('exceeds configured byte bound') + }) + + test('rejects cancelled or unbounded payload operations before a database write', async () => { + const store = payloadStore() + const controller = new AbortController() + controller.abort(new Error('cancelled publish')) + await expect( + store.publish({ + kind: 'outbox-data', + digest, + byteLength: '1', + bytes: (async function* () { + yield new Uint8Array([1]) + })(), + signal: controller.signal + }) + ).rejects.toThrow('cancelled publish') + const silent = new AbortController() + Object.defineProperty(silent.signal, 'aborted', { get: () => true }) + Object.defineProperty(silent.signal, 'reason', { get: () => undefined }) + await expect( + store.publish({ + kind: 'outbox-data', + digest, + byteLength: '1', + bytes: (async function* () { + yield new Uint8Array([1]) + })(), + signal: silent.signal + }) + ).rejects.toThrow('Mongo payload upload aborted') + const session = fakeSession() + const reference = { + scope, + payload: { kind: 'outbox-data' as const, digest }, + ownerKind: 'output' as const, + ownerId: 'owner', + slot: '0' + } + await expect(store.addReference(session, reference, { timeoutMS: 0 })).rejects.toThrow( + 'Invalid Mongo payload operation timeout' + ) + await expect(store.addReference(session, reference, { timeoutMS: 1.5 })).rejects.toThrow( + 'Invalid Mongo payload operation timeout' + ) + const aborted = new AbortController() + aborted.abort(new Error('cancelled reference')) + await expect( + store.addReference(session, reference, { signal: aborted.signal }) + ).rejects.toThrow('cancelled reference') + await expect(store.addReference(fakeSession(false), reference)).rejects.toThrow( + 'active transaction' + ) + await expect(store.claimGarbage(fakeSession(false), reference.payload)).rejects.toThrow( + 'active transaction' + ) + }) + + test('rejects payload references that violate owner, pin, or scope rules', async () => { + const store = payloadStore() + const session = fakeSession() + const payload = { kind: 'outbox-data' as const, digest } + await expect( + store.addReference(session, { + scope: { ...scope, nodeId: 'other' }, + payload, + ownerKind: 'output', + ownerId: 'owner', + slot: '0' + }) + ).rejects.toThrow('does not match this store') + await expect( + store.addReference(session, { + scope, + payload, + ownerKind: 'output', + ownerId: '', + slot: '0' + }) + ).rejects.toThrow('Invalid Mongo payload reference owner') + await expect( + store.addReference(session, { + scope, + payload, + ownerKind: 'output', + ownerId: 'owner', + slot: '' + }) + ).rejects.toThrow('Invalid Mongo payload reference owner') + await expect( + store.addReference(session, { + scope, + payload, + ownerKind: 'pin', + ownerId: 'owner', + slot: '0' + }) + ).rejects.toThrow('expiry is valid only for explicit pins') + await expect( + store.addReference(session, { + scope, + payload, + ownerKind: 'output', + ownerId: 'owner', + slot: '0', + expiresAt: new Date(Date.now() + 60_000) + }) + ).rejects.toThrow('expiry is valid only for explicit pins') + const frozen = payloadStore(dummyDb, { now: () => new Date('2020-01-01T00:00:00.000Z') }) + await expect( + frozen.addReference(session, { + scope, + payload, + ownerKind: 'pin', + ownerId: 'owner', + slot: '0', + expiresAt: new Date('2019-12-31T00:00:00.000Z') + }) + ).rejects.toThrow('pin must be unexpired') + }) + + test('rejects malformed manifest components before writing', async () => { + const store = payloadStore() + const session = fakeSession() + const payload = { kind: 'raw-transaction' as const, digest } + await expect( + store.addManifestComponent(session, { + manifestId: '', + ordinal: '0', + kind: 'raw-transaction', + payload + }) + ).rejects.toThrow('Invalid Mongo manifest component') + await expect( + store.addManifestComponent(session, { + manifestId: 'manifest', + ordinal: '01', + kind: 'raw-transaction', + payload + }) + ).rejects.toThrow('Invalid Mongo manifest component') + await expect( + store.addManifestComponent(session, { + manifestId: 'manifest', + ordinal: '0', + kind: 'outbox-data', + payload + }) + ).rejects.toThrow('kind mismatch') + await expect( + store.addManifestComponent(fakeSession(false), { + manifestId: 'manifest', + ordinal: '0', + kind: 'raw-transaction', + payload + }) + ).rejects.toThrow('active transaction') + }) + + test('treats a missing deleting row as an already-finished GC claim', async () => { + const store = payloadStore({ + collection: () => ({ + findOne: async () => null, + find: () => ({ toArray: async () => [] }), + updateOne: async () => ({ modifiedCount: 0 }) + }) + } as unknown as Db) + expect(await store.finishGarbage({ kind: 'outbox-data', digest })).toBe(false) + await expect(store.recoverUploads()).resolves.toBeUndefined() + }) + + test('idempotent and conflicting reference slots are decided from the existing row', async () => { + const payloadId = mongoRecordKey(mongoChainKey(scope), 'outbox-data', digest) + const refs: { findOne: jest.Mock } = { + findOne: jest.fn() + } + const payloads = { + updateOne: jest.fn(async () => ({ matchedCount: 0 })) + } + const db = { + collection: (name: string) => (name.includes('reference') ? refs : payloads) + } as unknown as Db + const store = payloadStore(db) + const session = fakeSession() + const reference = { + scope, + payload: { kind: 'outbox-data' as const, digest }, + ownerKind: 'output' as const, + ownerId: 'owner', + slot: '0' + } + refs.findOne.mockResolvedValueOnce({ payloadId }) + await expect(store.addReference(session, reference)).resolves.toBeUndefined() + refs.findOne.mockResolvedValueOnce({ payloadId: 'other' }) + await expect(store.addReference(session, reference)).rejects.toThrow( + 'already names different content' + ) + refs.findOne.mockResolvedValueOnce(null) + await expect(store.addReference(session, reference)).rejects.toThrow('not ready for reference') + }) + + test('release requires the named reference row to exist', async () => { + const store = payloadStore({ + collection: () => ({ + deleteOne: async () => ({ deletedCount: 0 }) + }) + } as unknown as Db) + await expect( + store.releaseReference(fakeSession(), { + scope, + payload: { kind: 'outbox-data', digest }, + ownerKind: 'output', + ownerId: 'owner', + slot: '0' + }) + ).rejects.toThrow('does not exist') + }) +}) + +describe('Mongo read guard input and sentinel guards', () => { + const key = 'output.0' + + test('swallows a duplicate-key race and then requires a matching sentinel', async () => { + const collection = { + updateOne: jest.fn(async () => { + throw { code: 11000 } + }), + findOne: jest.fn() + } + const guards = new MongoReadGuards({ collection: () => collection } as unknown as Db) + collection.findOne.mockResolvedValueOnce({ + network: scope.network, + genesisHash: scope.genesisHash, + nodeId: scope.nodeId, + key + }) + await expect(guards.initialize(scope, key)).resolves.toBeUndefined() + collection.findOne.mockResolvedValueOnce(null) + await expect(guards.initialize(scope, key)).rejects.toThrow( + 'Incompatible Mongo read guard sentinel' + ) + collection.updateOne.mockRejectedValueOnce({ code: 42 }) + await expect(guards.initialize(scope, key)).rejects.toMatchObject({ code: 42 }) + }) + + test('rejects an existing sentinel whose identity drifted', async () => { + const collection = { + updateOne: jest.fn(async () => ({})), + findOne: jest.fn(async () => ({ + network: 'other', + genesisHash: scope.genesisHash, + nodeId: scope.nodeId, + key + })) + } + const guards = new MongoReadGuards({ collection: () => collection } as unknown as Db) + await expect(guards.initialize(scope, key)).rejects.toThrow( + 'Incompatible Mongo read guard sentinel' + ) + }) + + test('check and changeVersion require a live transaction and matching version', async () => { + const collection = { + updateOne: jest.fn() + } + const guards = new MongoReadGuards({ collection: () => collection } as unknown as Db) + const read = { scope, key, expectedVersion: null } + await expect(guards.check(fakeSession(false), read)).rejects.toThrow('active transaction') + await expect(guards.changeVersion(fakeSession(false), read, 'v1')).rejects.toThrow( + 'active transaction' + ) + collection.updateOne.mockResolvedValueOnce({ matchedCount: 0 }) + await expect(guards.check(fakeSession(), read)).rejects.toBeInstanceOf( + MongoReadGuardConflictError + ) + collection.updateOne.mockResolvedValueOnce({ matchedCount: 0 }) + await expect(guards.changeVersion(fakeSession(), read, 'v1')).rejects.toBeInstanceOf( + MongoReadGuardConflictError + ) + collection.updateOne.mockResolvedValue({ matchedCount: 1 }) + await expect(guards.check(fakeSession(), read, { timeoutMS: 1 })).resolves.toBeUndefined() + await expect(guards.changeVersion(fakeSession(), read, 'v1')).resolves.toBeUndefined() + await expect(guards.changeVersion(fakeSession(), read, 'v'.repeat(257))).rejects.toThrow( + 'Invalid Mongo read guard version' + ) + await expect( + guards.check(fakeSession(), { ...read, expectedVersion: 'bad\u0000version' }) + ).rejects.toThrow('Invalid Mongo read guard version') + await expect( + guards.check(fakeSession(), { ...read, expectedVersion: '\ud800' }) + ).rejects.toThrow('Invalid Mongo read guard version') + }) + + test('rejects cancelled operations and non-integer timeouts', async () => { + const guards = new MongoReadGuards(dummyDb) + await expect(guards.initialize(scope, key, { timeoutMS: 0 })).rejects.toThrow( + 'Invalid Mongo read guard operation timeout' + ) + await expect(guards.initialize(scope, key, { timeoutMS: 1.5 })).rejects.toThrow( + 'Invalid Mongo read guard operation timeout' + ) + const controller = new AbortController() + Object.defineProperty(controller.signal, 'aborted', { get: () => true }) + Object.defineProperty(controller.signal, 'reason', { get: () => undefined }) + await expect(guards.initialize(scope, key, { signal: controller.signal })).rejects.toThrow( + 'Mongo read guard operation aborted' + ) + }) +}) + +describe('Mongo transaction runner input and budget guards', () => { + test('rejects constructor and call bounds that are not safe integers in range', () => { + expect(() => new MongoTransactionRunner(dummyDb, scope, { maxBodyAttempts: 0 })).toThrow( + 'Invalid Mongo transaction bound' + ) + expect(() => new MongoTransactionRunner(dummyDb, scope, { maxBodyAttempts: 11 })).toThrow( + 'Invalid Mongo transaction bound' + ) + expect(() => new MongoTransactionRunner(dummyDb, scope, { maxCommitAttempts: 1.5 })).toThrow( + 'Invalid Mongo transaction bound' + ) + expect(() => new MongoTransactionRunner(dummyDb, scope, { leaseMS: 0 })).toThrow( + 'Invalid Mongo transaction bound' + ) + expect(() => new MongoTransactionRunner(dummyDb, scope, { leaseMS: 60_001 })).toThrow( + 'Invalid Mongo transaction bound' + ) + expect(() => new MongoTransactionRunner(dummyDb, scope, { maxRetainedSessions: 0 })).toThrow( + 'Invalid Mongo transaction bound' + ) + expect(() => new MongoTransactionRunner(dummyDb, scope, { maxRetainedSessions: 1025 })).toThrow( + 'Invalid Mongo transaction bound' + ) + }) + + test('rejects malformed receipts, identities, and closed runners before storage work', async () => { + const runner = new MongoTransactionRunner(dummyDb, scope) + const body = jest.fn(async () => {}) + const valid = request() + const badDurability = structuredClone(valid) + ;(badDurability.receipt as { durability: string }).durability = 'best-effort' + await expect(runner.run(badDurability, body)).rejects.toThrow( + 'Invalid Mongo transaction receipt' + ) + const badSteak = structuredClone(valid) + badSteak.receipt.steak = '{' + await expect(runner.run(badSteak, body)).rejects.toThrow() + const illFormed = structuredClone(valid) + illFormed.receipt.steak = '\ud800' + await expect(runner.run(illFormed, body)).rejects.toThrow('Invalid Mongo transaction receipt') + const badIndex = structuredClone(valid) + badIndex.receipt.indexes = [{ target: '', state: 'pending' }] + await expect(runner.run(badIndex, body)).rejects.toThrow('Invalid Mongo transaction receipt') + const badState = structuredClone(valid) + badState.receipt.indexes = [{ target: 'lookup', state: 'missing' as 'pending' }] + await expect(runner.run(badState, body)).rejects.toThrow('Invalid Mongo transaction receipt') + const badPropagation = structuredClone(valid) + ;(badPropagation.receipt as { propagation: string }).propagation = 'done' + await expect(runner.run(badPropagation, body)).rejects.toThrow( + 'Invalid Mongo transaction receipt' + ) + const huge = structuredClone(valid) + huge.receipt.steak = JSON.stringify({ padding: 'x'.repeat(1_048_576) }) + await expect(runner.run(huge, body)).rejects.toThrow('too large') + const mismatchedPlan = structuredClone(valid) + mismatchedPlan.receipt.operationId = randomUUID() + expect(await runner.run(mismatchedPlan, body)).toEqual({ + state: 'rejected', + code: 'invalid-plan' + }) + const mismatchedDigest = structuredClone(valid) + mismatchedDigest.identity.contextDigest = '44'.repeat(32) + expect(await runner.run(mismatchedDigest, body)).toEqual({ + state: 'rejected', + code: 'digest-mismatch' + }) + await expect(runner.run(valid, body, { timeoutMS: 0 })).rejects.toThrow( + 'Invalid Mongo transaction bound' + ) + await expect(runner.run(valid, body, { timeoutMS: 50_001 })).rejects.toThrow( + 'Invalid Mongo transaction bound' + ) + await runner.close() + await expect(runner.run(valid, body)).rejects.toThrow('runner is closed') + expect(body).not.toHaveBeenCalled() + }) +}) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts index 7254ca0d7..c1b0bbd31 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts @@ -561,4 +561,166 @@ describe('MongoPayloadStore', () => { }) await session.endSession() }) + + test('cancels an in-flight stream after the first chunk has been accepted', async () => { + const content = Buffer.alloc(64 * 1024, 0x5e) + const hash = digest(content) + const controller = new AbortController() + const pending = store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: (async function* () { + yield content.subarray(0, 32 * 1024) + await delay(100) + yield content.subarray(32 * 1024) + })(), + signal: controller.signal + }) + await delay(20) + controller.abort(new Error('mid-stream cancel')) + await expect(pending).rejects.toThrow('mid-stream cancel') + }) + + test('rejects a non-byte stream, over-length stream, and iterator failure', async () => { + const hash = digest(Buffer.from('stream-guards')) + await expect( + store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: '1', + bytes: { + [Symbol.asyncIterator]() { + return { + next: async () => ({ done: false, value: 'nope' }), + return: async () => ({ done: true, value: undefined }) + } + } + } as AsyncIterable + }) + ).rejects.toThrow('non-byte chunk') + await expect( + store.publish({ + kind: 'outbox-data', + digest: digest(Buffer.from('too-long')), + byteLength: '1', + bytes: bytes(Buffer.from('ab')) + }) + ).rejects.toThrow('exceeds declared length') + await expect( + store.publish({ + kind: 'outbox-data', + digest: digest(Buffer.from('iterator-failed')), + byteLength: '1', + bytes: { + [Symbol.asyncIterator]() { + return { + next: async () => { + throw new Error('iterator-failed') + }, + return: async () => ({ done: true, value: undefined }) + } + } + } + }) + ).rejects.toThrow('iterator-failed') + }) + + test('reuses a reference slot for the same content and rejects a conflicting slot', async () => { + const content = Buffer.from('slot-reuse') + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const reference = { + scope: fixture.scope, + payload, + ownerKind: 'output' as const, + ownerId: 'slot-reuse', + slot: '0' + } + const session = fixture.client.startSession() + await session.withTransaction(async () => { + await store.addReference(session, reference) + await store.addReference(session, reference) + }) + await expect( + session.withTransaction(async () => { + await store.addReference(session, { + ...reference, + payload: { kind: 'locking-script', digest: hash } + }) + }) + ).rejects.toThrow('already names different content') + await session.withTransaction(async () => { + await store.releaseReference(session, reference) + }) + await expect( + session.withTransaction(async () => { + await store.releaseReference(session, reference) + }) + ).rejects.toThrow('does not exist') + await session.endSession() + }) + + test('rejects a reference to a payload that is not ready and reuses a matching manifest ordinal', async () => { + const missing = { kind: 'outbox-data' as const, digest: digest(Buffer.from('missing-ready')) } + const session = fixture.client.startSession() + await expect( + session.withTransaction(async () => { + await store.addReference(session, { + scope: fixture.scope, + payload: missing, + ownerKind: 'output', + ownerId: 'missing-ready', + slot: '0' + }) + }) + ).rejects.toThrow('not ready for reference') + const content = Buffer.from('manifest-ordinal') + const hash = digest(content) + const payload = { kind: 'raw-transaction' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const component = { + manifestId: 'manifest-ordinal', + ordinal: '0', + kind: 'raw-transaction' as const, + payload + } + await session.withTransaction(async () => { + await store.addManifestComponent(session, component) + await store.addManifestComponent(session, component) + }) + await fixture.db + .collection('overlay_manifest_components') + .updateOne({ manifestId: component.manifestId }, { $set: { kind: 'outbox-data' } }) + await expect( + session.withTransaction(async () => { + await store.addManifestComponent(session, component) + }) + ).rejects.toThrow('already names different content') + await session.endSession() + }) + + test('keeps an oversized inline payload from exceeding the BSON safety ceiling', async () => { + const tight = new MongoPayloadStore(fixture.db, fixture.scope, { + inlineCeilingBytes: 1024 * 1024 + }) + const content = Buffer.alloc(1024 * 1024, 0x21) + await expect( + tight.publish({ + kind: 'outbox-data', + digest: digest(content), + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024) + }) + ).rejects.toThrow('inline BSON document exceeds safety ceiling') + }, 30000) }) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoReadGuards.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoReadGuards.test.ts index f79d3eb7f..2419b331e 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoReadGuards.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoReadGuards.test.ts @@ -97,6 +97,36 @@ describe('Mongo read guards', () => { } }) + test('changeVersion conflicts when the persisted version no longer matches', async () => { + const session = fixture.client.startSession() + try { + session.startTransaction() + await guards.changeVersion(session, absentRead(), 'v1') + await session.commitTransaction() + } finally { + await session.endSession() + } + const stale = fixture.client.startSession() + try { + stale.startTransaction() + await expect(guards.changeVersion(stale, absentRead(), 'v2')).rejects.toBeInstanceOf( + MongoReadGuardConflictError + ) + await stale.abortTransaction() + } finally { + await stale.endSession() + } + }) + + test('refuses a sentinel whose stored identity no longer matches the requested key', async () => { + await fixture.db + .collection(MongoCollectionNames.readGuards) + .updateOne({ key }, { $set: { network: 'other-network' } }) + await expect(guards.initialize(fixture.scope, key)).rejects.toThrow( + 'Incompatible Mongo read guard sentinel' + ) + }) + test('rejects malformed read inputs and bounded operation controls', async () => { await expect(guards.initialize(fixture.scope, '', { timeoutMS: 1 })).rejects.toThrow( 'Invalid Mongo read guard key' diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts index aa09e6b1c..89fb0611d 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts @@ -18,6 +18,11 @@ describe('Mongo schema codecs', () => { expect('bootstrapMongoOverlay' in overlay).toBe(false) expect('MongoPayloadStore' in overlay).toBe(false) expect('MongoTransactionRunner' in overlay).toBe(false) + const mongo = await import('../../storage/mongo.js') + expect(mongo.bootstrapMongoOverlay).toEqual(expect.any(Function)) + expect(mongo.MongoPayloadStore).toEqual(expect.any(Function)) + expect(mongo.MongoReadGuards).toEqual(expect.any(Function)) + expect(mongo.MongoTransactionRunner).toEqual(expect.any(Function)) }) test('uses framed values rather than separator-concatenated keys', () => { @@ -313,6 +318,37 @@ describe('Mongo schema bootstrap', () => { ).rejects.toThrow() }) + test('refuses a ledger whose schema fingerprint is not this Overlay schema', async () => { + const database = fixture.client.db(`overlay_s02_ledger_${Date.now()}`) + await bootstrapMongoOverlay(database, fixture.scope) + await database + .collection(MongoCollectionNames.schema) + .updateOne( + { _id: mongoNodeKey(fixture.scope) }, + { $set: { schemaFingerprint: 'aa'.repeat(32) } } + ) + await expect(bootstrapMongoOverlay(database, fixture.scope)).rejects.toThrow( + 'Incompatible Mongo Overlay schema ledger' + ) + }) + + test('refuses an existing collection whose collation is not Overlay simple', async () => { + const definition = MongoCollectionDefinitions.find( + item => item.name === MongoCollectionNames.outputs + ) + if (definition === undefined) throw new Error('Missing outputs schema definition') + const database = fixture.client.db(`overlay_s02_collation_${Date.now()}`) + await database.createCollection(MongoCollectionNames.outputs, { + validator: definition.validator, + validationLevel: 'strict', + validationAction: 'error', + collation: { locale: 'en' } + }) + await expect(bootstrapMongoOverlay(database, fixture.scope)).rejects.toThrow( + `Incompatible Mongo Overlay validator for ${MongoCollectionNames.outputs}` + ) + }) + test('rejects a pre-existing partial index whose predicate has drifted', async () => { const definition = MongoCollectionDefinitions.find( item => item.name === MongoCollectionNames.payloadReferences diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts index 39a8ddb62..03cc1828d 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts @@ -51,6 +51,32 @@ describe('Mongo transaction boundary on three data-bearing WiredTiger members', expect(getAdmissionStorage(runner)).toBeUndefined() }) + test('rejects a committed row whose receipt is missing or identity-corrupt', async () => { + const input = request() + await runner.run(input, async () => {}) + const collection = fixture.db.collection(MongoCollectionNames.submissionOperations) + await collection.updateOne({ _id: operationId(input) }, { $unset: { receipt: '' } }) + await expect(runner.reconcile(input.key)).rejects.toThrow('Committed Mongo operation has no receipt') + await collection.updateOne( + { _id: operationId(input) }, + { $set: { receipt: new Binary(Buffer.from(JSON.stringify({ ...input.receipt, operationId: randomUUID() }))) } } + ) + await expect(runner.reconcile(input.key)).rejects.toThrow('Corrupt Mongo operation receipt identity') + }) + + test('cannot close while a trusted body is still running', async () => { + const input = request() + let resume!: () => void + const barrier = new Promise(resolve => { resume = resolve }) + let entered!: () => void + const started = new Promise(resolve => { entered = resolve }) + const pending = runner.run(input, async () => { entered(); await barrier }) + await started + await expect(runner.close()).rejects.toThrow('during a call') + resume() + expect((await pending).state).toBe('committed') + }) + test('saves exact receipt with effects and replays it after runner restart', async () => { const input = request() let bodies = 0 diff --git a/packages/overlays/overlay/src/storage/Storage.ts b/packages/overlays/overlay/src/storage/Storage.ts index 80b08be8a..1afb8e9a4 100644 --- a/packages/overlays/overlay/src/storage/Storage.ts +++ b/packages/overlays/overlay/src/storage/Storage.ts @@ -262,3 +262,8 @@ export interface Storage { */ getLastInteraction: (host: string, topic: string) => Promise } + +/** True when the optional admission field is present. Protocol detection is separate. */ +export function storageHasAdmission(storage: Storage): boolean { + return storage.admission !== undefined +} From 30f0398fae7ee701073b34e598eaf9e7d8bd7b47 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 00:35:03 -0500 Subject: [PATCH 085/168] fix(sonar): use toHaveLength in LookupResolver discovery tests --- .../src/overlay-tools/__tests/LookupResolver.dynamic.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts index ed59565af..c6d78fa33 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts @@ -1058,7 +1058,7 @@ describe('LookupResolver dynamic discovery', () => { await jest.runAllTimersAsync() await pending } - expect(lookup.mock.calls.filter(([url]) => url === tracker).length).toBe(missingFields.length) + expect(lookup.mock.calls.filter(([url]) => url === tracker)).toHaveLength(missingFields.length) }) it('rejects a deadline outside the accepted range', async () => { From b03108f3af5a9d32c18ce72ed5a34498181a76ac Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 00:35:05 -0500 Subject: [PATCH 086/168] fix(overlay): retry Mongo write conflicts on claim as well as body Concurrent snapshot spends can throw WriteConflict from findOneAndUpdate outside the transaction body. Treat those as transient at the run loop. --- .../storage/mongo/MongoTransactionRunner.ts | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts b/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts index 8d3add746..439128967 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoTransactionRunner.ts @@ -85,8 +85,16 @@ function hasLabel(error: unknown, label: string): boolean { function isTransientTransactionError(error: unknown): boolean { if (hasLabel(error, 'TransientTransactionError')) return true - if (typeof error === 'object' && error !== null && 'code' in error && error.code === 112) return true - return error instanceof Error && error.message.includes('Write conflict') + if (typeof error !== 'object' || error === null) return false + const code = 'code' in error ? error.code : undefined + if (code === 112 || code === '112') return true + if ('codeName' in error && error.codeName === 'WriteConflict') return true + const text = [ + error instanceof Error ? error.message : '', + 'errmsg' in error ? String(error.errmsg) : '', + 'cause' in error && error.cause instanceof Error ? error.cause.message : '' + ].join(' ') + return text.includes('Write conflict') } function duplicateKey(error: unknown): boolean { @@ -292,8 +300,12 @@ export class MongoTransactionRunner { this.calls += 1 try { for (let bodyIndex = 0; bodyIndex < this.maxBodyAttempts; bodyIndex += 1) { - const completed = await this.runBodyAttempt(id, request, receipt, body, budget, bodyIndex) - if (completed !== undefined) return completed + try { + const completed = await this.runBodyAttempt(id, request, receipt, body, budget, bodyIndex) + if (completed !== undefined) return completed + } catch (error) { + if (!isTransientTransactionError(error) || bodyIndex + 1 >= this.maxBodyAttempts) throw error + } } throw new Error('Mongo transaction attempt limit reached') } finally { From 582a1e988c03032de2b03b615d8d7aae692312c0 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 00:55:03 -0500 Subject: [PATCH 087/168] test(sdk): cover chain tracker and evidence helpers for patch coverage --- packages/sdk/docs/reference/transaction.md | 16 ++- packages/sdk/docs/transaction-evidence.md | 3 +- packages/sdk/src/transaction/ChainTracker.ts | 13 ++ .../transaction/__tests/ChainTracker.test.ts | 29 ++++ .../__tests/TransactionEvidence.test.ts | 130 ++++++++++++++++++ .../TransactionEvidenceCoordinator.test.ts | 46 +++++++ packages/sdk/src/transaction/index.ts | 1 + .../Wallet.identityVerification.test.ts | 71 ++++++++++ .../__tests/ChaintracksChainTracker.test.ts | 104 ++++++++++++++ 9 files changed, 411 insertions(+), 2 deletions(-) create mode 100644 packages/sdk/src/transaction/__tests/ChainTracker.test.ts create mode 100644 packages/sdk/src/transaction/__tests/TransactionEvidence.test.ts diff --git a/packages/sdk/docs/reference/transaction.md b/packages/sdk/docs/reference/transaction.md index 7d74d8b00..4b6b7db65 100644 --- a/packages/sdk/docs/reference/transaction.md +++ b/packages/sdk/docs/reference/transaction.md @@ -3063,7 +3063,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | [evidenceError](#function-evidenceerror) | [scriptVerificationBackend](#function-scriptverificationbackend) | | [evidenceScriptScope](#function-evidencescriptscope) | [unregisterScriptVerificationBackend](#function-unregisterscriptverificationbackend) | | [executeNodejsRequest](#function-executenodejsrequest) | [withEvidenceScriptWork](#function-withevidencescriptwork) | -| [isBroadcastFailure](#function-isbroadcastfailure) | | +| [isBroadcastFailure](#function-isbroadcastfailure) | [isChainTracker](#function-ischaintracker) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -3208,6 +3208,20 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( --- +### Function: isChainTracker + +Convenience type guard for a `ChainTracker` implementation. Optional verification-context methods are not required. + +```ts +export function isChainTracker(value: unknown): value is ChainTracker +``` + +See also: [ChainTracker](./transaction.md#interface-chaintracker) + +Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) + +--- + ### Function: parseEvidence ```ts diff --git a/packages/sdk/docs/transaction-evidence.md b/packages/sdk/docs/transaction-evidence.md index 238bfa4fb..3a57f6329 100644 --- a/packages/sdk/docs/transaction-evidence.md +++ b/packages/sdk/docs/transaction-evidence.md @@ -60,7 +60,8 @@ unobserved ABA transition back to the same tip, and a token over multiple sources is not an atomic multi-source snapshot. Canonical roots and observed heights are therefore still rechecked on every use, including positive-cache reuse. The required tracker methods accept optional abort signals; existing -implementations may ignore them. +implementations may ignore them. `isChainTracker(value)` is a runtime type +guard for those required methods; optional context methods are not required. The limits in [`TransactionEvidence.ts`](../src/transaction/TransactionEvidence.ts) are local admission and memory policy, not consensus limits. Defaults are: diff --git a/packages/sdk/src/transaction/ChainTracker.ts b/packages/sdk/src/transaction/ChainTracker.ts index 9d2d78f12..6ac0ca4f6 100644 --- a/packages/sdk/src/transaction/ChainTracker.ts +++ b/packages/sdk/src/transaction/ChainTracker.ts @@ -41,3 +41,16 @@ export default interface ChainTracker { */ getVerificationContextToken?: (signal?: AbortSignal) => Promise } + +/** + * Convenience type guard for a `ChainTracker` implementation. + * Optional verification-context methods are not required. + */ +export function isChainTracker(value: unknown): value is ChainTracker { + if (value == null || typeof value !== 'object') return false + const tracker = value as Partial + return ( + typeof tracker.isValidRootForHeight === 'function' && + typeof tracker.currentHeight === 'function' + ) +} diff --git a/packages/sdk/src/transaction/__tests/ChainTracker.test.ts b/packages/sdk/src/transaction/__tests/ChainTracker.test.ts new file mode 100644 index 000000000..59946d7fe --- /dev/null +++ b/packages/sdk/src/transaction/__tests/ChainTracker.test.ts @@ -0,0 +1,29 @@ +import ChainTracker, { isChainTracker } from '../ChainTracker' + +describe('isChainTracker', () => { + const tracker: ChainTracker = { + isValidRootForHeight: async () => true, + currentHeight: async () => 0 + } + + it('accepts an object with the required methods', () => { + expect(isChainTracker(tracker)).toBe(true) + if (isChainTracker(tracker)) { + expect(typeof tracker.isValidRootForHeight).toBe('function') + expect(typeof tracker.currentHeight).toBe('function') + } + }) + + it('rejects nullish and non-object values', () => { + expect(isChainTracker(null)).toBe(false) + expect(isChainTracker(undefined)).toBe(false) + expect(isChainTracker('tracker')).toBe(false) + expect(isChainTracker(1)).toBe(false) + }) + + it('rejects objects missing a required method', () => { + expect(isChainTracker({})).toBe(false) + expect(isChainTracker({ isValidRootForHeight: async () => true })).toBe(false) + expect(isChainTracker({ currentHeight: async () => 0 })).toBe(false) + }) +}) diff --git a/packages/sdk/src/transaction/__tests/TransactionEvidence.test.ts b/packages/sdk/src/transaction/__tests/TransactionEvidence.test.ts new file mode 100644 index 000000000..34cdbe7de --- /dev/null +++ b/packages/sdk/src/transaction/__tests/TransactionEvidence.test.ts @@ -0,0 +1,130 @@ +import MerklePath from '../MerklePath' +import Transaction from '../Transaction' +import { + assertEvidenceUnchanged, + defaultTransactionEvidenceLimits, + parseEvidence, + TransactionEvidenceError +} from '../TransactionEvidence' +import P2PKH from '../../script/templates/P2PKH' +import PrivateKey from '../../primitives/PrivateKey' +import Script from '../../script/Script' + +const height = 700_000 + +async function signedChild(): Promise { + const key = new PrivateKey(46) + const p2pkh = new P2PKH() + const source = new Transaction() + source.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + source.addOutput({ satoshis: 10, lockingScript: p2pkh.lock(key.toAddress()) }) + source.merklePath = new MerklePath(height, [ + [ + { offset: 0, hash: source.id('hex'), txid: true }, + { offset: 1, duplicate: true } + ] + ]) + const tx = new Transaction() + tx.addInput({ + sourceTransaction: source, + sourceOutputIndex: 0, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + tx.addOutput({ satoshis: 4, lockingScript: p2pkh.lock(key.toAddress()) }) + await tx.sign() + return tx +} + +function expectInvalid(evidence: { beef: number[]; outputIndex: number; txid?: string }): void { + expect(() => parseEvidence(evidence, defaultTransactionEvidenceLimits)).toThrow( + TransactionEvidenceError + ) +} + +describe('parseEvidence', () => { + it('rejects malformed snapshots before parsing', () => { + expectInvalid({ beef: 'nope' as unknown as number[], outputIndex: 0 }) + expectInvalid({ beef: [], outputIndex: 0 }) + expectInvalid({ beef: [256], outputIndex: 0 }) + expectInvalid({ beef: [-1], outputIndex: 0 }) + expectInvalid({ beef: [1.5], outputIndex: 0 }) + expectInvalid({ beef: [1], outputIndex: -1 }) + expectInvalid({ beef: [1], outputIndex: 1.5 }) + }) + + it('rejects an output index past the selected transaction', async () => { + const tx = await signedChild() + expectInvalid({ beef: tx.toBEEF(), outputIndex: tx.outputs.length }) + }) + + it('rejects a mismatched source TXID and an unconfirmed zero-input leaf', async () => { + const tx = await signedChild() + tx.inputs[0].sourceTXID = '11'.repeat(32) + expectInvalid({ beef: tx.toBEEF(), outputIndex: 0 }) + + const leaf = new Transaction() + leaf.addOutput({ satoshis: 1, lockingScript: Script.fromASM('OP_TRUE') }) + expectInvalid({ beef: leaf.toBEEF(), outputIndex: 0 }) + }) + + it('walks a shared unconfirmed ancestor once and fences mutated owned bytes', async () => { + const key = new PrivateKey(47) + const p2pkh = new P2PKH() + const confirmed = new Transaction() + confirmed.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + confirmed.addOutput({ satoshis: 20, lockingScript: p2pkh.lock(key.toAddress()) }) + confirmed.merklePath = new MerklePath(height, [ + [ + { offset: 0, hash: confirmed.id('hex'), txid: true }, + { offset: 1, duplicate: true } + ] + ]) + const ancestor = new Transaction() + ancestor.addInput({ + sourceTransaction: confirmed, + sourceOutputIndex: 0, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + ancestor.addOutput({ satoshis: 9, lockingScript: p2pkh.lock(key.toAddress()) }) + ancestor.addOutput({ satoshis: 9, lockingScript: p2pkh.lock(key.toAddress()) }) + await ancestor.sign() + const children: Transaction[] = [] + for (const outputIndex of [0, 1]) { + const child = new Transaction() + child.addInput({ + sourceTransaction: ancestor, + sourceOutputIndex: outputIndex, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + child.addOutput({ satoshis: 8, lockingScript: p2pkh.lock(key.toAddress()) }) + await child.sign() + children.push(child) + } + const joined = new Transaction() + for (const sourceTransaction of children) { + joined.addInput({ + sourceTransaction, + sourceOutputIndex: 0, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + } + joined.addOutput({ satoshis: 15, lockingScript: p2pkh.lock(key.toAddress()) }) + await joined.sign() + + const candidate = parseEvidence( + { beef: joined.toBEEF(), outputIndex: 0 }, + defaultTransactionEvidenceLimits + ) + expect(candidate.txid).toBe(joined.id('hex')) + candidate.tx.outputs[0].satoshis++ + expect(() => assertEvidenceUnchanged(candidate)).toThrow(TransactionEvidenceError) + }) +}) diff --git a/packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts b/packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts index 456956973..5eed91b31 100644 --- a/packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts +++ b/packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts @@ -1082,4 +1082,50 @@ describe('TransactionEvidenceCoordinator', () => { await expectCode(subject.verify({ beef: evidence, outputIndex: 0 }), 'limit') } ) + + it('rejects invalid constructor context and limits, then a disposed or pre-aborted verify', async () => { + const { tracker, evidence } = await fixture() + expect( + () => + new TransactionEvidenceCoordinator({ + chainTracker: null as unknown as ChainTracker, + chainNamespace: 'local-canonical-chain', + policyId: 'p2pkh-consensus' + }) + ).toThrow(TransactionEvidenceError) + expect( + () => + new TransactionEvidenceCoordinator({ + chainTracker: tracker, + chainNamespace: '', + policyId: 'p2pkh-consensus' + }) + ).toThrow(TransactionEvidenceError) + expect(() => coordinator(tracker, { consumers: 0 })).toThrow(TransactionEvidenceError) + + const disposed = coordinator(tracker) + disposed.dispose() + await expectCode(disposed.verify({ beef: evidence, outputIndex: 0 }), 'disposed') + + const abort = new AbortController() + abort.abort() + await expectCode( + coordinator(tracker).verify({ beef: evidence, outputIndex: 0 }, { signal: abort.signal }), + 'cancelled' + ) + }) + + it('rejects a second distinct transaction once the pending-transaction limit is full', async () => { + const { tracker, firstEvidence, secondEvidence } = await sharedAncestorFixture() + const release = deferred() + const entered = deferred() + tracker.gate = release.promise + tracker.onRootCall = () => entered.resolve() + const limited = coordinator(tracker, { pendingTransactions: 1 }) + const first = limited.verify({ beef: firstEvidence, outputIndex: 0 }) + await entered.promise + await expectCode(limited.verify({ beef: secondEvidence, outputIndex: 0 }), 'limit') + release.resolve() + await expect(first).resolves.toMatchObject({ outputIndex: 0 }) + }) }) diff --git a/packages/sdk/src/transaction/index.ts b/packages/sdk/src/transaction/index.ts index accd1dbcd..eaa617460 100644 --- a/packages/sdk/src/transaction/index.ts +++ b/packages/sdk/src/transaction/index.ts @@ -10,6 +10,7 @@ export type { default as TransactionOutput } from './TransactionOutput.js' export type { Broadcaster, BroadcastFailure, BroadcastResponse } from './Broadcaster.js' export { isBroadcastResponse, isBroadcastFailure } from './Broadcaster.js' export type { default as ChainTracker } from './ChainTracker.js' +export { isChainTracker } from './ChainTracker.js' export { TransactionEvidenceCoordinator } from './TransactionEvidenceCoordinator.js' export type { TransactionEvidenceContext, diff --git a/packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts b/packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts index a42bd6102..1f25583f4 100644 --- a/packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts +++ b/packages/wallet/wallet-toolbox/src/__tests/Wallet.identityVerification.test.ts @@ -5,6 +5,7 @@ import { MerklePath, PrivateKey, Transaction, + TransactionEvidenceError, Validation, VerifiableCertificate } from '@bsv/sdk' @@ -12,6 +13,7 @@ import { Wallet } from '../Wallet' import { WalletSettingsManager } from '../WalletSettingsManager' import { WalletStorageManager } from '../storage/WalletStorageManager' import { WalletServices } from '../sdk/WalletServices.interfaces' +import { IdentityEvidenceVerifier } from '../utility/identityUtils' import { createIdentityVerificationFixture, IdentityVerificationFixture @@ -235,4 +237,73 @@ describe('Wallet final identity verification and compatibility', () => { await expect(wallet.discoverByAttributes({ attributes: {}, offset: -1 })).rejects.toThrow() expect(query).toHaveBeenCalledTimes(2) }) + + it('returns no overlay certificates when services are omitted without forceRefresh', async () => { + const { wallet, query } = walletFor(fixture) + wallet.services = undefined + await expect(wallet.discoverByIdentityKey({ identityKey: fixture.certificate.subject })).resolves.toEqual({ + totalCertificates: 0, + certificates: [] + }) + expect(query).not.toHaveBeenCalled() + }) + + it('returns no overlay certificates after destroy closes identity evidence', async () => { + const { wallet, query } = walletFor(fixture) + await wallet.destroy() + await expect(wallet.discoverByIdentityKey({ identityKey: fixture.certificate.subject })).resolves.toEqual({ + totalCertificates: 0, + certificates: [] + }) + expect(query).not.toHaveBeenCalled() + }) + + it('drops an in-flight overlay lookup after destroy closes identity evidence', async () => { + const { wallet, query } = walletFor(fixture) + let resolveLookup: ((value: LookupAnswer) => void) | undefined + const started = new Promise(resolve => { + query.mockImplementation( + async () => + await new Promise(resolveAnswer => { + resolve() + resolveLookup = resolveAnswer + }) + ) + }) + const pending = wallet.discoverByIdentityKey({ identityKey: fixture.certificate.subject }) + await started + await wallet.destroy() + resolveLookup!({ type: 'output-list', outputs: [{ beef: fixture.certificateBEEF, outputIndex: 0 }] }) + await expect(pending).resolves.toEqual({ totalCertificates: 0, certificates: [] }) + }) + + it('deletes overlay evidence when parseResults throws a bounded limit', async () => { + const { wallet, query } = walletFor(fixture) + const parse = jest + .spyOn(IdentityEvidenceVerifier.prototype, 'parse') + .mockRejectedValue(new TransactionEvidenceError('limit')) + const args = { identityKey: fixture.certificate.subject } + try { + await expect(wallet.discoverByIdentityKey(args)).rejects.toMatchObject({ code: 'limit' }) + } finally { + parse.mockRestore() + } + await expect(wallet.discoverByIdentityKey(args)).resolves.toMatchObject({ totalCertificates: 1 }) + expect(query).toHaveBeenCalledTimes(2) + }) + + it('expires overlay evidence through its scheduled prune', async () => { + jest.useFakeTimers({ now: Date.now() }) + try { + const { wallet, query } = walletFor(fixture) + const args = { identityKey: fixture.certificate.subject } + await expect(wallet.discoverByIdentityKey(args)).resolves.toMatchObject({ totalCertificates: 1 }) + expect(query).toHaveBeenCalledTimes(1) + await jest.advanceTimersByTimeAsync(2 * 60 * 1000 + 1) + await expect(wallet.discoverByIdentityKey(args)).resolves.toMatchObject({ totalCertificates: 1 }) + expect(query).toHaveBeenCalledTimes(2) + } finally { + jest.useRealTimers() + } + }) }) diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts index 32d23a42e..34ef1bbfa 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts @@ -229,6 +229,110 @@ describe('ChaintracksChaintracker tests', () => { await expect(tracker.getVerificationContextToken()).rejects.toThrow('Method not implemented.') expect(subscribe).toHaveBeenCalled() }) + + test('ignores a chaintracks setter that does not change the provider', async () => { + const provider = makeChaintracksClient([HEADER_877599]) + const tracker = new ChaintracksChainTracker('main', provider) + const context = tracker.getVerificationContext() + tracker.chaintracks = provider + expect(tracker.getVerificationContext()).toBe(context) + }) + + test('obtains a token without reorg events and rejects a promised-events client without subscribeReorgs', async () => { + const silent = makeChaintracksClient([]) + silent.findChainTipHash = jest.fn(async () => 'aa'.repeat(32)) + const silentTracker = new ChaintracksChainTracker('main', silent) + await expect(silentTracker.getVerificationContextToken()).resolves.toContain('aa'.repeat(32)) + + const promised = makeChaintracksClient([]) + promised.supportsReorgEvents = true + promised.findChainTipHash = jest.fn(async () => 'aa'.repeat(32)) + const promisedTracker = new ChaintracksChainTracker('main', promised) + await expect(promisedTracker.getVerificationContextToken()).rejects.toThrow( + 'promised reorg events but subscribeReorgs is not implemented' + ) + }) + + test('shares one in-flight reorg registration across concurrent token lookups', async () => { + let release: ((value: string) => void) | undefined + const pending = new Promise(resolve => { + release = resolve + }) + const provider = makeChaintracksClient([]) + provider.findChainTipHash = jest.fn(async () => 'aa'.repeat(32)) + provider.subscribeReorgs = jest.fn(async () => await pending) + provider.unsubscribe = jest.fn(async () => true) + const tracker = new ChaintracksChainTracker('main', provider) + + const first = tracker.getVerificationContextToken() + const second = tracker.getVerificationContextToken() + release!('shared-subscription') + await expect(Promise.all([first, second])).resolves.toEqual([ + expect.stringContaining('aa'.repeat(32)), + expect.stringContaining('aa'.repeat(32)) + ]) + expect(provider.subscribeReorgs).toHaveBeenCalledTimes(1) + }) + + test('rejects height and token lookups when the provider changes while they are pending', async () => { + let tracker: ChaintracksChainTracker + const heightProvider = makeChaintracksClient([]) + heightProvider.getPresentHeight.mockImplementation(async () => { + tracker.chaintracks = makeChaintracksClient([]) + return 950000 + }) + tracker = new ChaintracksChainTracker('main', heightProvider, { maxRetries: 1, retryDelayMs: 0 }) + await expect(tracker.currentHeight()).rejects.toThrow('provider changed during height lookup') + + const tokenProvider = makeChaintracksClient([]) + tokenProvider.findChainTipHash = jest.fn(async () => { + tracker.chaintracks = makeChaintracksClient([]) + return 'aa'.repeat(32) + }) + tracker = new ChaintracksChainTracker('main', tokenProvider) + await expect(tracker.getVerificationContextToken()).rejects.toThrow( + 'provider changed during canonical token lookup' + ) + }) + + test('aborts height lookup with the signal reason after the provider returns', async () => { + const abort = new AbortController() + const provider = makeChaintracksClient([]) + provider.getPresentHeight.mockImplementation(async () => { + abort.abort(new Error('height-aborted')) + return 950000 + }) + const tracker = new ChaintracksChainTracker('main', provider) + await expect(tracker.currentHeight(abort.signal)).rejects.toThrow('height-aborted') + await expect(tracker.currentHeight({ aborted: true } as AbortSignal)).rejects.toThrow( + 'Chaintracks verification aborted' + ) + }) + + test('prunes diagnostic cache entries older than five minutes', async () => { + jest.useFakeTimers({ now: Date.now() }) + try { + const older = HEADER_877599 + const newer = { + ...HEADER_877599, + height: 877600, + merkleRoot: '22'.repeat(32), + hash: '33'.repeat(32) + } + const provider = { + getPresentHeight: jest.fn(async () => 950000), + findHeaderForHeight: jest.fn(async (height: number) => (height === older.height ? older : newer)) + } + const tracker = new ChaintracksChainTracker('main', provider as any, { maxRetries: 1, retryDelayMs: 0 }) + await expect(tracker.isValidRootForHeight(older.merkleRoot, older.height)).resolves.toBe(true) + await jest.advanceTimersByTimeAsync(5 * 60 * 1000 + 1) + await expect(tracker.isValidRootForHeight(newer.merkleRoot, newer.height)).resolves.toBe(true) + expect(tracker.cache[older.height]).toBeUndefined() + expect(tracker.cache[newer.height]).toBe(newer.merkleRoot) + } finally { + jest.useRealTimers() + } + }) }) async function testChaintracksChaintracker(chain: sdk.Chain) { From 9ba25270ea3f360173df0ab916030e6ff8cb60f8 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 00:56:37 -0500 Subject: [PATCH 088/168] fix(overlay): retry admission commits on Mongo write conflicts Concurrent snapshot spends still escaped the transaction-runner retry. commitAdmission now retries WriteConflict up to eight times. --- .../storage/mongo/MongoAdmissionStorage.ts | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts index 9d8e0d635..c9a7c6e62 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts @@ -44,6 +44,18 @@ import { type MongoTransactionOptions } from './MongoTransactionRunner.js' +function isMongoWriteConflict(error: unknown): boolean { + if (typeof error !== 'object' || error === null) return false + const code = 'code' in error ? error.code : undefined + if (code === 112 || code === '112') return true + if ('codeName' in error && error.codeName === 'WriteConflict') return true + const text = [ + error instanceof Error ? error.message : '', + 'errmsg' in error ? String(error.errmsg) : '' + ].join(' ') + return text.includes('Write conflict') +} + export interface MongoEnlistedLookupIndex { readonly protocol: 'overlay-mongo-index-v1' readonly target: string @@ -214,18 +226,22 @@ export class MongoAdmissionStorage implements AdmissionStorage { } await this.prepareReadGuards(plan) await this.publishHistoryUpdatePayloads(plan) - try { - const result = await this.runner.run( - { key: plan.key, identity: plan.identity, receipt }, - async context => { - await this.applyPlan(context, plan) - } - ) - if (result.state !== 'pending') return result - return await this.waitForPending(plan.key, result.attemptId) - } catch (error) { - return this.asResult(error) + for (let attempt = 0; attempt < 8; attempt += 1) { + try { + const result = await this.runner.run( + { key: plan.key, identity: plan.identity, receipt }, + async context => { + await this.applyPlan(context, plan) + } + ) + if (result.state !== 'pending') return result + return await this.waitForPending(plan.key, result.attemptId) + } catch (error) { + if (isMongoWriteConflict(error) && attempt < 7) continue + return this.asResult(error) + } } + return { state: 'rejected', code: 'spend-conflict' } } private async waitForPending( From 8a3ec78e6909800c14cde9914d95313d781b859c Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 00:59:07 -0500 Subject: [PATCH 089/168] fix(sonar): extract admission write-conflict retry helper S3776: commitAdmission was 21 after the retry loop. The loop now lives in runAdmissionWithWriteConflictRetry. --- .../overlay/src/storage/mongo/MongoAdmissionStorage.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts index c9a7c6e62..0895fa620 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoAdmissionStorage.ts @@ -226,6 +226,13 @@ export class MongoAdmissionStorage implements AdmissionStorage { } await this.prepareReadGuards(plan) await this.publishHistoryUpdatePayloads(plan) + return await this.runAdmissionWithWriteConflictRetry(plan, receipt) + } + + private async runAdmissionWithWriteConflictRetry( + plan: AdmissionCommit, + receipt: ReturnType + ): Promise { for (let attempt = 0; attempt < 8; attempt += 1) { try { const result = await this.runner.run( From d63befdd96a870bf4b3ae6a5d2087342e5c9fc65 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 01:34:36 -0500 Subject: [PATCH 090/168] test(overlay): cover mongo admission commit for patch coverage --- .../src/__tests/Engine.admission.test.ts | 105 ++++- .../src/__tests/EngineAdmission.test.ts | 414 ++++++++++++++++++ .../src/__tests/StorageContract.test.ts | 8 + .../__tests/mongo/MongoAdmissionPlan.test.ts | 265 +++++++++++ .../__tests/mongo/MongoAdmissionRetry.test.ts | 305 +++++++++++++ .../mongo/MongoAdmissionStorage.test.ts | 226 +++++++++- 6 files changed, 1321 insertions(+), 2 deletions(-) create mode 100644 packages/overlays/overlay/src/__tests/EngineAdmission.test.ts create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoAdmissionRetry.test.ts diff --git a/packages/overlays/overlay/src/__tests/Engine.admission.test.ts b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts index 481c64de8..6952a3460 100644 --- a/packages/overlays/overlay/src/__tests/Engine.admission.test.ts +++ b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts @@ -2,6 +2,7 @@ import { Engine } from '../Engine.js' import type { LookupService } from '../LookupService.js' import type { TopicManager } from '../TopicManager.js' import type { Storage } from '../storage/Storage.js' +import type { OverlayAdmissionHost } from '../EngineAdmission.js' import type { AdmissionCommit, AdmissionReceipt, @@ -48,7 +49,8 @@ function receiptFor(plan: AdmissionCommit): AdmissionReceipt { describe('Engine admission submit', () => { let mockTopicManager: TopicManager let mockLookupService: LookupService - let mockStorage: Storage & { admission: AdmissionStorage; admissionScope: StorageScope } + let mockStorage: Storage & + OverlayAdmissionHost & { admission: AdmissionStorage; admissionScope: StorageScope } let commitAdmission: jest.Mock let saved: AdmissionReceipt | undefined @@ -236,4 +238,105 @@ describe('Engine admission submit', () => { expect(plan.decisions[0].applied.firstSeenHeight).toBe('800000') expect(plan.decisions[0].applied.block).toBeUndefined() }) + + test('returns STEAK without commit when no topic is accepted', async () => { + mockTopicManager.identifyAdmissibleOutputs = jest.fn(async () => ({ + outputsToAdmit: [], + coinsToRetain: [] + })) + const onReady = jest.fn() + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker + ) + const steak = await engine.submit({ beef: exampleBeef, topics: ['Hello'] }, onReady) + expect(steak).toEqual({ Hello: { outputsToAdmit: [], coinsToRetain: [] } }) + expect(commitAdmission).not.toHaveBeenCalled() + expect(onReady).toHaveBeenCalledWith(steak) + }) + + test('uses host payload, fence, and enlisted index hooks', async () => { + const publishAdmissionPayload = jest.fn( + async (input: { kind: string; bytes: Uint8Array; txid?: string }) => ({ + kind: input.kind, + digest: 'ab'.repeat(32), + byteLength: String(input.bytes.byteLength) + }) + ) + const getHistoryFence = jest.fn(async () => ({ + chainEpoch: '1', + topicHistoryGeneration: '2' + })) + mockStorage.publishAdmissionPayload = publishAdmissionPayload + mockStorage.enlistedIndexTargets = () => ['Hello'] + mockStorage.getHistoryFence = getHistoryFence + mockStorage.findOutput = jest.fn(async () => ({ + txid: + exampleTX.inputs[0].sourceTXID ?? + '3ecead27a44d013ad1aae40038acbb1883ac9242406808bb4667c15b4f164eac', + outputIndex: exampleTX.inputs[0].sourceOutputIndex, + outputScript: [], + satoshis: 1, + topic: 'Hello', + spent: false, + outputsConsumed: [], + consumedBy: [] + })) + mockTopicManager.identifyAdmissibleOutputs = jest.fn(async () => ({ + outputsToAdmit: [0], + coinsToRetain: [0] + })) + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker + ) + await engine.submit({ beef: exampleBeef, topics: ['Hello'] }, undefined, 'current-tx', [7, 8]) + const submitted = commitAdmission.mock.calls[0][0] as AdmissionCommit + expect(publishAdmissionPayload).toHaveBeenCalled() + expect(getHistoryFence).toHaveBeenCalledWith('Hello') + expect(submitted.decisions[0].expectedHistory).toEqual({ + chainEpoch: '1', + topicHistoryGeneration: '2' + }) + expect(submitted.decisions[0].spends).toHaveLength(1) + expect(submitted.outbox.some(intent => intent.target === 'Hello')).toBe(false) + expect(submitted.identity.contextDigest.length).toBe(64) + }) + + test('surfaces admission rejection and recovers a pending receipt', async () => { + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker + ) + commitAdmission.mockResolvedValueOnce({ state: 'rejected', code: 'spend-conflict' }) + await expect(engine.submit({ beef: exampleBeef, topics: ['Hello'] })).rejects.toThrow( + 'Overlay admission rejected: spend-conflict' + ) + + commitAdmission + .mockResolvedValueOnce({ state: 'rejected', code: 'read-conflict' }) + .mockImplementation(async (plan: AdmissionCommit) => { + saved = receiptFor(plan) + return { state: 'committed' as const, receipt: saved } + }) + await expect(engine.submit({ beef: exampleBeef, topics: ['Hello'] })).resolves.toEqual( + expect.objectContaining({ Hello: expect.objectContaining({ outputsToAdmit: [0] }) }) + ) + + commitAdmission.mockResolvedValue({ state: 'pending', attemptId: 'lost-ack' }) + const lastPlan = commitAdmission.mock.calls[commitAdmission.mock.calls.length - 1]?.[0] as + AdmissionCommit | undefined + mockStorage.admission.reconcileAdmission = jest.fn(async () => ({ + state: 'committed' as const, + receipt: + saved ?? receiptFor(lastPlan ?? (commitAdmission.mock.calls[0][0] as AdmissionCommit)) + })) + await expect(engine.submit({ beef: exampleBeef, topics: ['Hello'] })).resolves.toBeDefined() + }) }) diff --git a/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts b/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts new file mode 100644 index 000000000..728dd4c4d --- /dev/null +++ b/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts @@ -0,0 +1,414 @@ +import { Transaction } from '@bsv/sdk' +import type { LookupService } from '../LookupService.js' +import type { Output } from '../Output.js' +import { + asStorageUint64, + type AdmissionCommit, + type AdmissionStorage, + type StorageScope +} from '../storage/AdmissionStorage.js' +import { + buildOverlayAdmissionPlan, + getOverlayAdmissionHost, + overlayAdmissionContextDigest, + overlayAdmissionMode, + overlayAdmissionOperationId, + waitForAdmissionReceipt, + type OverlayAdmissionHost +} from '../EngineAdmission.js' + +const BRC62Hex = + '0100beef01fe636d0c0007021400fe507c0c7aa754cef1f7889d5fd395cf1f785dd7de98eed895dbedfe4e5bc70d1502ac4e164f5bc16746bb0868404292ac8318bbac3800e4aad13a014da427adce3e010b00bc4ff395efd11719b277694cface5aa50d085a0bb81f613f70313acd28cf4557010400574b2d9142b8d28b61d88e3b2c3f44d858411356b49a28a4643b6d1a6a092a5201030051a05fc84d531b5d250c23f4f886f6812f9fe3f402d61607f977b4ecd2701c19010000fd781529d58fc2523cf396a7f25440b409857e7e221766c57214b1d38c7b481f01010062f542f45ea3660f86c013ced80534cb5fd4c19d66c56e7e8c5d4bf2d40acc5e010100b121e91836fd7cd5102b654e9f72f3cf6fdbfd0b161c53a9c54b12c841126331020100000001cd4e4cac3c7b56920d1e7655e7e260d31f29d9a388d04910f1bbd72304a79029010000006b483045022100e75279a205a547c445719420aa3138bf14743e3f42618e5f86a19bde14bb95f7022064777d34776b05d816daf1699493fcdf2ef5a5ab1ad710d9c97bfb5b8f7cef3641210263e2dee22b1ddc5e11f6fab8bcd2378bdd19580d640501ea956ec0e786f93e76ffffffff013e660000000000001976a9146bfd5c7fbe21529d45803dbcf0c87dd3c71efbc288ac0000000001000100000001ac4e164f5bc16746bb0868404292ac8318bbac3800e4aad13a014da427adce3e000000006a47304402203a61a2e931612b4bda08d541cfb980885173b8dcf64a3471238ae7abcd368d6402204cbf24f04b9aa2256d8901f0ed97866603d2be8324c2bfb7a37bf8fc90edd5b441210263e2dee22b1ddc5e11f6fab8bcd2378bdd19580d640501ea956ec0e786f93e76ffffffff013c660000000000001976a9146bfd5c7fbe21529d45803dbcf0c87dd3c71efbc288ac0000000000' + +const exampleTX = Transaction.fromHexBEEF(BRC62Hex) +const exampleBeef = exampleTX.toBEEF() +const exampleTxid = exampleTX.id('hex') +const previousTxid = + exampleTX.inputs[0].sourceTXID ?? exampleTX.inputs[0].sourceTransaction?.id('hex') +if (typeof previousTxid !== 'string') + throw new Error('expected the example transaction to name its source') + +const admissionScope: StorageScope = { + network: 'testnet', + genesisHash: '11'.repeat(32), + nodeId: 'engine-node' +} + +const lookupServices = { Hello: {} as LookupService } + +const previousOutput: Output = { + txid: previousTxid, + outputIndex: exampleTX.inputs[0].sourceOutputIndex, + outputScript: [], + satoshis: 1, + topic: 'Hello', + spent: false, + outputsConsumed: [], + consumedBy: [] +} + +function host(overrides: Partial = {}): OverlayAdmissionHost { + return { + admission: { + protocol: 'overlay-admission-v1', + commitAdmission: jest.fn(), + reconcileAdmission: jest.fn() + }, + admissionScope, + ...overrides + } +} + +function validation( + overrides: Partial<{ + topic: string + isDupe: boolean + previousCoins: number[] + previousOutputs: Array + outputsToAdmit: number[] + coinsToRetain: number[] + coinsRemoved: number[] + }> = {} +) { + return { + topic: overrides.topic ?? 'Hello', + isDupe: overrides.isDupe ?? false, + previousCoins: overrides.previousCoins ?? [], + previousOutputs: overrides.previousOutputs ?? [], + admissibleOutputs: { + outputsToAdmit: overrides.outputsToAdmit ?? [0], + coinsToRetain: overrides.coinsToRetain ?? [], + coinsRemoved: overrides.coinsRemoved + } + } +} + +describe('Engine overlay admission helpers', () => { + test('getOverlayAdmissionHost requires v1 admission and a hex genesis', () => { + expect(getOverlayAdmissionHost(undefined)).toBeUndefined() + expect(getOverlayAdmissionHost(null)).toBeUndefined() + expect(getOverlayAdmissionHost({})).toBeUndefined() + const admission = { + protocol: 'overlay-admission-v1' as const, + commitAdmission: jest.fn(), + reconcileAdmission: jest.fn() + } + expect(getOverlayAdmissionHost({ admission })).toBeUndefined() + expect( + getOverlayAdmissionHost({ + admission, + admissionScope: { ...admissionScope, genesisHash: 'zz'.repeat(32) } + }) + ).toBeUndefined() + expect( + getOverlayAdmissionHost({ + admission, + admissionScope: { network: 1, genesisHash: admissionScope.genesisHash, nodeId: 'n' } + }) + ).toBeUndefined() + const publishAdmissionPayload = jest.fn() + const enlistedIndexTargets = jest.fn(() => ['Hello']) + const getHistoryFence = jest.fn() + const found = getOverlayAdmissionHost({ + admission, + admissionScope, + publishAdmissionPayload, + enlistedIndexTargets, + getHistoryFence + }) + expect(found?.admission).toBe(admission) + expect(found?.admissionScope).toEqual(admissionScope) + expect(found?.admissionScope).not.toBe(admissionScope) + expect(found?.publishAdmissionPayload).toBe(publishAdmissionPayload) + expect(found?.enlistedIndexTargets).toBe(enlistedIndexTargets) + expect(found?.getHistoryFence).toBe(getHistoryFence) + }) + + test('operation ids stay raw when short and hash when oversized or ill-formed', () => { + const short = overlayAdmissionOperationId('live', exampleTxid, ['Hello']) + expect(short).toBe(`submit:live:${exampleTxid}:Hello`) + const hashed = overlayAdmissionOperationId('historical', exampleTxid, ['t'.repeat(600)]) + expect(hashed.startsWith('submit:')).toBe(true) + expect(hashed).not.toContain('t'.repeat(600)) + expect(overlayAdmissionOperationId('live', exampleTxid, ['\ud800']).startsWith('submit:')).toBe( + true + ) + expect(overlayAdmissionMode('current-tx')).toBe('live') + expect(overlayAdmissionMode('historical-tx')).toBe('historical') + expect(overlayAdmissionMode('historical-tx-no-spv')).toBe('historical') + expect(overlayAdmissionContextDigest()).not.toBe(overlayAdmissionContextDigest([1, 2])) + }) + + test('buildOverlayAdmissionPlan throws when no topic is accepted', async () => { + await expect( + buildOverlayAdmissionPlan({ + host: host(), + tx: exampleTX, + txid: exampleTxid, + beef: exampleBeef, + topics: ['Hello'], + mode: 'live', + validations: [validation({ outputsToAdmit: [], coinsToRetain: [] })], + failedTopics: new Set(), + lookupServices, + includePropagation: true + }) + ).rejects.toThrow('Overlay admission plan has no topics') + }) + + test('builds spends, evictions, enlisted outbox, and published payloads', async () => { + const publishAdmissionPayload = jest.fn(async input => ({ + kind: input.kind, + digest: overlayAdmissionContextDigest(Array.from(input.bytes)), + byteLength: asStorageUint64(String(input.bytes.byteLength)) + })) + const getHistoryFence = jest.fn(async () => ({ + chainEpoch: asStorageUint64('4'), + topicHistoryGeneration: asStorageUint64('5') + })) + const plan = await buildOverlayAdmissionPlan({ + host: host({ + publishAdmissionPayload, + getHistoryFence, + enlistedIndexTargets: () => ['Hello'] + }), + tx: exampleTX, + txid: exampleTxid, + beef: exampleBeef, + topics: ['Hello'], + mode: 'live', + offChainValues: [9], + validations: [ + validation({ + previousCoins: [0], + previousOutputs: [previousOutput, null], + coinsToRetain: [0], + outputsToAdmit: [0, 0, 99] + }), + validation({ topic: 'Failed', outputsToAdmit: [0] }) + ], + failedTopics: new Set(['Failed']), + lookupServices: { Hello: {} as LookupService, Other: {} as LookupService }, + includePropagation: true, + applied: { + firstSeenHeight: 10, + blockHeight: 11, + blockHash: 'aa'.repeat(32), + blockIndex: 2, + merkleRoot: 'bb'.repeat(32) + } + }) + expect(plan.identity.mode).toBe('live') + expect(plan.identity.contextDigest).toBe(overlayAdmissionContextDigest([9])) + expect(plan.decisions).toHaveLength(1) + expect(plan.decisions[0].expectedHistory).toEqual({ + chainEpoch: '4', + topicHistoryGeneration: '5' + }) + expect(plan.decisions[0].spends).toEqual([ + { + outpoint: { txid: previousTxid, outputIndex: asStorageUint64('0') }, + expectedVersion: '1', + spender: exampleTxid + } + ]) + expect(plan.decisions[0].evictions).toEqual([]) + expect(plan.decisions[0].edges).toHaveLength(2) + expect(plan.decisions[0].outputs).toHaveLength(2) + expect(plan.decisions[0].applied.block).toEqual({ + height: '11', + hash: 'aa'.repeat(32), + index: '2', + merkleRoot: 'bb'.repeat(32) + }) + expect(plan.outbox.some(intent => intent.target === 'Hello')).toBe(false) + expect(plan.outbox.some(intent => intent.kind === 'propagation')).toBe(true) + expect(publishAdmissionPayload).toHaveBeenCalled() + expect(getHistoryFence).toHaveBeenCalledWith('Hello') + }) + + test('historical plans omit propagation and attach merkle proof metadata', async () => { + const proven = exampleTX.inputs[0].sourceTransaction + if (proven === undefined) throw new Error('expected a proven ancestor') + const plan = await buildOverlayAdmissionPlan({ + host: host(), + tx: proven, + txid: proven.id('hex'), + beef: proven.toBEEF(), + topics: ['Hello'], + mode: 'historical', + validations: [validation({ isDupe: true, outputsToAdmit: [] })], + failedTopics: new Set(), + lookupServices, + includePropagation: true + }) + expect(plan.identity.mode).toBe('historical') + expect(plan.outbox.some(intent => intent.kind === 'propagation')).toBe(false) + expect(plan.decisions[0].applied.proof?.kind).toBe('merkle-path') + expect(plan.decisions[0].applied.firstSeenHeight).toBeDefined() + }) + + test('marks unmatched previous coins stale and retains consumed coins', async () => { + const stale = await buildOverlayAdmissionPlan({ + host: host(), + tx: exampleTX, + txid: exampleTxid, + beef: exampleBeef, + topics: ['Hello'], + mode: 'live', + validations: [ + validation({ + previousCoins: [0], + previousOutputs: [previousOutput], + coinsToRetain: [] + }) + ], + failedTopics: new Set(), + lookupServices, + includePropagation: false + }) + expect(stale.decisions[0].evictions).toEqual([ + { + txid: previousTxid, + outputIndex: asStorageUint64(String(exampleTX.inputs[0].sourceOutputIndex)) + } + ]) + expect(JSON.parse(stale.steak).Hello.coinsRemoved).toEqual([0]) + }) + + test('skips previous coins whose source txid cannot be recovered', async () => { + const tx = { + toBinary: () => exampleTX.toBinary(), + merklePath: undefined, + outputs: exampleTX.outputs, + inputs: [{ sourceOutputIndex: 0 }] + } as unknown as Transaction + const plan = await buildOverlayAdmissionPlan({ + host: host(), + tx, + txid: exampleTxid, + beef: exampleBeef, + topics: ['Hello'], + mode: 'live', + validations: [validation({ previousCoins: [0], previousOutputs: [previousOutput] })], + failedTopics: new Set(), + lookupServices, + includePropagation: false + }) + expect(plan.decisions[0].evictions).toEqual([]) + expect(plan.decisions[0].edges).toEqual([]) + expect(plan.outbox.some(intent => intent.kind === 'propagation')).toBe(false) + }) +}) + +describe('waitForAdmissionReceipt', () => { + const plan = { + key: { + scope: admissionScope, + operationId: 'op', + semanticDigest: 'aa'.repeat(32) + } + } as AdmissionCommit + + test('returns a committed result and retries a read-conflict rebuild', async () => { + const receipt = { + operationId: 'op', + semanticDigest: 'aa'.repeat(32), + durability: 'atomic-local' as const, + steak: '{}', + indexes: [], + propagation: 'not-requested' as const + } + const admission: AdmissionStorage = { + protocol: 'overlay-admission-v1', + commitAdmission: jest + .fn() + .mockResolvedValueOnce({ state: 'rejected', code: 'read-conflict' }) + .mockResolvedValueOnce({ state: 'committed', receipt }), + reconcileAdmission: jest.fn() + } + const rebuild = jest.fn(async () => plan) + await expect(waitForAdmissionReceipt(admission, plan, rebuild)).resolves.toEqual({ + state: 'committed', + receipt + }) + expect(rebuild).toHaveBeenCalledTimes(1) + }) + + test('throws a non-retryable rejection and a rejected reconcile', async () => { + await expect( + waitForAdmissionReceipt( + { + protocol: 'overlay-admission-v1', + commitAdmission: jest.fn(async () => ({ state: 'rejected', code: 'spend-conflict' })), + reconcileAdmission: jest.fn() + }, + plan, + async () => plan + ) + ).rejects.toThrow('Overlay admission rejected: spend-conflict') + + await expect( + waitForAdmissionReceipt( + { + protocol: 'overlay-admission-v1', + commitAdmission: jest.fn(async () => ({ state: 'pending', attemptId: 'a1' })), + reconcileAdmission: jest.fn(async () => ({ state: 'rejected', code: 'invalid-plan' })) + }, + plan, + async () => plan + ) + ).rejects.toThrow('Overlay admission rejected: invalid-plan') + }) + + test('reconciles a lost ack, rebuilds after abort, and times out while pending', async () => { + const receipt = { + operationId: 'op', + semanticDigest: 'aa'.repeat(32), + durability: 'atomic-local' as const, + steak: '{}', + indexes: [], + propagation: 'pending' as const + } + await expect( + waitForAdmissionReceipt( + { + protocol: 'overlay-admission-v1', + commitAdmission: jest.fn(async () => ({ state: 'pending', attemptId: 'lost' })), + reconcileAdmission: jest.fn(async () => ({ state: 'committed', receipt })) + }, + plan, + async () => plan + ) + ).resolves.toEqual({ state: 'committed', receipt }) + + const rebuild = jest.fn(async () => plan) + await expect( + waitForAdmissionReceipt( + { + protocol: 'overlay-admission-v1', + commitAdmission: jest + .fn() + .mockResolvedValueOnce({ state: 'pending', attemptId: 'aborted' }) + .mockResolvedValueOnce({ state: 'committed', receipt }), + reconcileAdmission: jest.fn(async () => ({ state: 'aborted' })) + }, + plan, + rebuild + ) + ).resolves.toEqual({ state: 'committed', receipt }) + expect(rebuild).toHaveBeenCalledTimes(1) + + await expect( + waitForAdmissionReceipt( + { + protocol: 'overlay-admission-v1', + commitAdmission: jest.fn(async () => ({ state: 'pending', attemptId: 'stuck' })), + reconcileAdmission: jest.fn(async () => ({ state: 'pending', attemptId: 'stuck' })) + }, + plan, + async () => plan + ) + ).rejects.toThrow('Overlay admission commit is pending') + }) +}) diff --git a/packages/overlays/overlay/src/__tests/StorageContract.test.ts b/packages/overlays/overlay/src/__tests/StorageContract.test.ts index e613d7315..f8f5d38d3 100644 --- a/packages/overlays/overlay/src/__tests/StorageContract.test.ts +++ b/packages/overlays/overlay/src/__tests/StorageContract.test.ts @@ -4,6 +4,7 @@ import type { Knex } from 'knex' import { KnexStorage } from '../storage/knex/KnexStorage.js' import { admissionSemanticDigest, + asStorageUint64, getAdmissionStorage, isReplaySafeProjection, parseStorageOutputIndex, @@ -93,6 +94,13 @@ describe('S01 portable persistence contract', () => { expect(() => parseStorageUint64(1 as unknown as string)).toThrow() }) + test('asStorageUint64 returns the canonical decimal after validation', () => { + expect(asStorageUint64('0')).toBe('0') + expect(asStorageUint64('18446744073709551615')).toBe('18446744073709551615') + expect(() => asStorageUint64('01')).toThrow('Invalid storage uint64') + expect(() => asStorageUint64('-1')).toThrow('Invalid storage uint64') + }) + test.each(fixture.outputIndices)('exact output index: "$value"', ({ value, valid }) => { if (valid) expect(String(parseStorageOutputIndex(value))).toBe(value) else expect(() => parseStorageOutputIndex(value)).toThrow() diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts new file mode 100644 index 000000000..94c08689c --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts @@ -0,0 +1,265 @@ +import { + admissionSemanticDigest, + type AdmissionCommit, + type AdmissionPayloadRef +} from '../../storage/AdmissionStorage.js' +import { + AdmissionRejectedError, + admissionPlanPayloads, + admissionReceiptFor, + lookupOutboxIntents, + propagationOutboxIntents, + rejectAdmission, + samePayload, + sameScope, + validateAdmissionPlan +} from '../../storage/mongo/MongoAdmissionPlan.js' +import { admissionPlan } from '../admission/AdmissionStorageContract.js' +import { referenceScope } from '../admission/ReferenceAdmissionStorage.js' + +const clone = (value: T): T => structuredClone(value) +const hash = (character: string): string => character.repeat(64) + +function bind(plan: AdmissionCommit): AdmissionCommit { + plan.key.semanticDigest = admissionSemanticDigest(plan.identity) + return plan +} + +describe('Mongo admission plan validation', () => { + test('accepts the contract plan and classifies outbox intents', () => { + const plan = admissionPlan() + expect(validateAdmissionPlan(plan)).toBeUndefined() + expect(lookupOutboxIntents(plan).map(intent => intent.target)).toEqual(['ls_contract']) + expect(propagationOutboxIntents(plan).map(intent => intent.kind)).toEqual(['propagation']) + expect(admissionPlanPayloads(plan).length).toBeGreaterThan(plan.payloads.length) + }) + + test('sameScope and samePayload compare every identity field', () => { + expect(sameScope(referenceScope, { ...referenceScope })).toBe(true) + expect(sameScope(referenceScope, { ...referenceScope, nodeId: 'other' })).toBe(false) + const payload: AdmissionPayloadRef = { + digest: hash('a'), + byteLength: '100', + kind: 'raw-transaction' + } + expect(samePayload(payload, payload)).toBe(true) + expect(samePayload(undefined, payload)).toBe(false) + expect(samePayload({ ...payload, kind: 'locking-script' }, payload)).toBe(false) + expect(samePayload({ ...payload, byteLength: '99' }, payload)).toBe(false) + }) + + test('rejects a digest that does not bind the identity or scope', () => { + const mismatched = clone(admissionPlan()) + mismatched.key.semanticDigest = hash('0') + expect(validateAdmissionPlan(mismatched)).toBe('digest-mismatch') + + const scoped = clone(admissionPlan()) + scoped.key.scope = { ...referenceScope, nodeId: 'other-node' } + expect(validateAdmissionPlan(scoped)).toBe('digest-mismatch') + + const invalid = clone(admissionPlan()) + invalid.identity.txid = 'not-a-hash' + expect(validateAdmissionPlan(invalid)).toBe('digest-mismatch') + }) + + test('rejects historical propagation and inconsistent topic sets', () => { + const historical = bind(clone(admissionPlan('historical-operation'))) + historical.identity.mode = 'historical' + bind(historical) + expect(validateAdmissionPlan(historical)).toBe('invalid-plan') + + const missingDecision = bind(clone(admissionPlan('missing-decision'))) + missingDecision.identity.topics.push({ topic: 'tm_other', policyId: 'policy-2' }) + bind(missingDecision) + expect(validateAdmissionPlan(missingDecision)).toBe('invalid-plan') + + const unknownTopic = clone(admissionPlan('unknown-topic')) + unknownTopic.decisions[0].topic = 'tm_other' + expect(validateAdmissionPlan(unknownTopic)).toBe('invalid-plan') + + const duplicateEvents = clone(admissionPlan('duplicate-events')) + duplicateEvents.outbox[1].eventId = duplicateEvents.outbox[0].eventId + expect(validateAdmissionPlan(duplicateEvents)).toBe('invalid-plan') + }) + + test('rejects payloads that are not ready for reference', () => { + const badDigest = clone(admissionPlan('payload-digest')) + badDigest.payloads[0] = { ...badDigest.payloads[0], digest: 'zz' } + expect(validateAdmissionPlan(badDigest)).toBe('payload-not-ready') + + const badLength = clone(admissionPlan('payload-length')) + badLength.payloads[0] = { ...badLength.payloads[0], byteLength: '01' } + expect(validateAdmissionPlan(badLength)).toBe('payload-not-ready') + }) + + test('rejects STEAK that is not bound to the admitted outputs', () => { + const notJson = clone(admissionPlan('steak-json')) + notJson.steak = '{' + expect(validateAdmissionPlan(notJson)).toBe('invalid-plan') + + const illFormed = clone(admissionPlan('steak-ill')) + illFormed.steak = '\ud800' + expect(validateAdmissionPlan(illFormed)).toBe('invalid-plan') + + const arraySteak = clone(admissionPlan('steak-array')) + arraySteak.steak = '[]' + expect(validateAdmissionPlan(arraySteak)).toBe('invalid-plan') + + const scalar = clone(admissionPlan('steak-scalar')) + scalar.steak = '1' + expect(validateAdmissionPlan(scalar)).toBe('invalid-plan') + + const retainType = clone(admissionPlan('steak-retain')) + retainType.steak = JSON.stringify({ + tm_contract: { outputsToAdmit: [0], coinsToRetain: 'nope' } + }) + expect(validateAdmissionPlan(retainType)).toBe('invalid-plan') + + const removedType = clone(admissionPlan('steak-removed')) + removedType.steak = JSON.stringify({ + tm_contract: { outputsToAdmit: [0], coinsRemoved: 1 } + }) + expect(validateAdmissionPlan(removedType)).toBe('invalid-plan') + + const fractional = clone(admissionPlan('steak-fraction')) + fractional.steak = JSON.stringify({ tm_contract: { outputsToAdmit: [0.5] } }) + expect(validateAdmissionPlan(fractional)).toBe('invalid-plan') + + const negative = clone(admissionPlan('steak-negative')) + negative.steak = JSON.stringify({ tm_contract: { outputsToAdmit: [-1] } }) + expect(validateAdmissionPlan(negative)).toBe('invalid-plan') + + const overflow = clone(admissionPlan('steak-overflow')) + overflow.steak = JSON.stringify({ tm_contract: { outputsToAdmit: [4294967296] } }) + expect(validateAdmissionPlan(overflow)).toBe('invalid-plan') + + const mismatch = clone(admissionPlan('steak-mismatch')) + mismatch.steak = JSON.stringify({ + tm_contract: { outputsToAdmit: [7], coinsToRetain: [], coinsRemoved: [] } + }) + expect(validateAdmissionPlan(mismatch)).toBe('invalid-plan') + + const notString = clone(admissionPlan('steak-type')) + ;(notString as { steak: unknown }).steak = { tm_contract: { outputsToAdmit: [0] } } + expect(validateAdmissionPlan(notString)).toBe('invalid-plan') + }) + + test('rejects malformed outbox intents', () => { + const emptyId = clone(admissionPlan('outbox-empty-id')) + emptyId.outbox[0].eventId = '' + expect(validateAdmissionPlan(emptyId)).toBe('invalid-plan') + + const illId = clone(admissionPlan('outbox-ill-id')) + illId.outbox[0].eventId = '\ud800' + expect(validateAdmissionPlan(illId)).toBe('invalid-plan') + + const emptyTarget = clone(admissionPlan('outbox-empty-target')) + emptyTarget.outbox[0].target = '' + expect(validateAdmissionPlan(emptyTarget)).toBe('invalid-plan') + + const illTarget = clone(admissionPlan('outbox-ill-target')) + illTarget.outbox[0].target = '\ud800' + expect(validateAdmissionPlan(illTarget)).toBe('invalid-plan') + + const kind = clone(admissionPlan('outbox-kind')) + ;(kind.outbox[0] as { kind: string }).kind = 'index' + expect(validateAdmissionPlan(kind)).toBe('invalid-plan') + }) + + test('rejects malformed spends, edges, outputs, and applied history', () => { + const spendTxid = clone(admissionPlan('spend-txid')) + spendTxid.decisions[0].spends[0].outpoint.txid = 'nope' + expect(validateAdmissionPlan(spendTxid)).toBe('invalid-plan') + + const spender = clone(admissionPlan('spend-spender')) + spender.decisions[0].spends[0].spender = hash('1') + expect(validateAdmissionPlan(spender)).toBe('invalid-plan') + + const eviction = clone(admissionPlan('eviction')) + eviction.decisions[0].evictions = [{ txid: 'nope', outputIndex: '0' }] + expect(validateAdmissionPlan(eviction)).toBe('invalid-plan') + + const edge = clone(admissionPlan('edge')) + edge.decisions[0].edges[0].source.txid = 'nope' + expect(validateAdmissionPlan(edge)).toBe('invalid-plan') + + const outputTxid = clone(admissionPlan('output-txid')) + outputTxid.decisions[0].outputs[0].txid = hash('7') + expect(validateAdmissionPlan(outputTxid)).toBe('invalid-plan') + + const satoshis = clone(admissionPlan('satoshis')) + satoshis.decisions[0].outputs[0].satoshis = '01' + expect(validateAdmissionPlan(satoshis)).toBe('invalid-plan') + + const scriptRange = clone(admissionPlan('script-range')) + scriptRange.decisions[0].outputs[0].script.offset = '20' + scriptRange.decisions[0].outputs[0].script.byteLength = '10' + expect(validateAdmissionPlan(scriptRange)).toBe('invalid-plan') + + const appliedTxid = clone(admissionPlan('applied-txid')) + appliedTxid.decisions[0].applied.txid = hash('7') + expect(validateAdmissionPlan(appliedTxid)).toBe('invalid-plan') + + const firstSeen = clone(admissionPlan('first-seen')) + firstSeen.decisions[0].applied.firstSeenHeight = '01' + expect(validateAdmissionPlan(firstSeen)).toBe('invalid-plan') + + const proof = clone(admissionPlan('applied-proof')) + proof.decisions[0].applied.proof = { digest: 'zz', byteLength: '1', kind: 'merkle-path' } + expect(validateAdmissionPlan(proof)).toBe('payload-not-ready') + + const block = clone(admissionPlan('applied-block')) + block.decisions[0].applied.block = { + height: '1', + hash: 'not-a-hash', + index: '0', + merkleRoot: hash('4') + } + expect(validateAdmissionPlan(block)).toBe('invalid-plan') + + const history = clone(admissionPlan('history')) + history.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '3', + affectedFromHeight: '99' + } + expect(validateAdmissionPlan(history)).toBe('invalid-plan') + + const historyType = clone(admissionPlan('history-type')) + historyType.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '4', + affectedFromHeight: '01' + } + expect(validateAdmissionPlan(historyType)).toBe('invalid-plan') + }) + + test('receipts mark enlisted indexes visible and reject overlapping lookup targets', () => { + const plan = admissionPlan() + expect(admissionReceiptFor(plan, [])).toEqual({ + operationId: plan.key.operationId, + semanticDigest: plan.key.semanticDigest, + durability: 'atomic-local', + steak: plan.steak, + indexes: [{ target: 'ls_contract', state: 'pending' }], + propagation: 'pending' + }) + expect(admissionReceiptFor(plan, ['ls_enlisted']).indexes).toEqual([ + { target: 'ls_enlisted', state: 'visible' }, + { target: 'ls_contract', state: 'pending' } + ]) + expect(() => admissionReceiptFor(plan, ['ls_contract'])).toThrow(AdmissionRejectedError) + try { + admissionReceiptFor(plan, ['ls_contract']) + } catch (error) { + expect(error).toMatchObject({ name: 'AdmissionRejectedError', code: 'invalid-plan' }) + } + + const noPropagation = clone(plan) + noPropagation.outbox = noPropagation.outbox.filter(intent => intent.kind !== 'propagation') + expect(admissionReceiptFor(noPropagation, []).propagation).toBe('not-requested') + }) + + test('rejectAdmission throws a coded admission error', () => { + expect(() => rejectAdmission('digest-mismatch')).toThrow(AdmissionRejectedError) + expect(() => rejectAdmission('spend-conflict')).toThrow('spend-conflict') + }) +}) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionRetry.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionRetry.test.ts new file mode 100644 index 000000000..16d6656c9 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionRetry.test.ts @@ -0,0 +1,305 @@ +import type { Db } from 'mongodb' +import { + admissionSemanticDigest, + asStorageUint64, + type AdmissionCommit, + type AdmissionReceipt +} from '../../storage/AdmissionStorage.js' +import { MongoReadGuardConflictError } from '../../storage/mongo/MongoReadGuards.js' +import { + AdmissionRejectedError, + admissionReceiptFor +} from '../../storage/mongo/MongoAdmissionPlan.js' +import { MongoAdmissionStorage } from '../../storage/mongo/MongoAdmissionStorage.js' +import { MongoOverlayStorage } from '../../storage/mongo/MongoOverlayStorage.js' +import type { MongoTransactionRunner } from '../../storage/mongo/MongoTransactionRunner.js' +import { admissionPlan } from '../admission/AdmissionStorageContract.js' +import { referenceScope } from '../admission/ReferenceAdmissionStorage.js' + +const dummyDb = { collection: () => ({}) } as unknown as Db + +const clone = (value: T): T => structuredClone(value) + +function plan(): AdmissionCommit { + const next = clone(admissionPlan('retry')) + next.decisions[0].reads = [] + return next +} + +function receiptFor(value: AdmissionCommit): AdmissionReceipt { + return admissionReceiptFor(value, []) +} + +function runner( + overrides: Partial> = {} +): MongoTransactionRunner { + return { + reconcile: + overrides.reconcile ?? (async () => ({ state: 'pending' as const, attemptId: 'unlocated' })), + run: + overrides.run ?? + (async () => ({ + state: 'committed' as const, + receipt: receiptFor(plan()) + })), + close: async () => {} + } as unknown as MongoTransactionRunner +} + +function storage(overrides: ConstructorParameters[2] = {}) { + return new MongoAdmissionStorage(dummyDb, referenceScope, { + runner: runner(), + readGuards: { initialize: async () => {} } as never, + ...overrides + }) +} + +describe('Mongo admission commit guards and write-conflict retry', () => { + test('constructor rejects invalid and duplicate enlisted indexes', () => { + expect( + () => + new MongoAdmissionStorage(dummyDb, referenceScope, { + enlistedIndexes: [ + { protocol: 'other' as 'overlay-mongo-index-v1', target: 'ls', apply: async () => {} } + ] + }) + ).toThrow('Invalid enlisted Mongo lookup index') + expect( + () => + new MongoAdmissionStorage(dummyDb, referenceScope, { + enlistedIndexes: [ + { protocol: 'overlay-mongo-index-v1', target: '', apply: async () => {} } + ] + }) + ).toThrow('Invalid enlisted Mongo lookup index') + expect( + () => + new MongoAdmissionStorage(dummyDb, referenceScope, { + enlistedIndexes: [ + { protocol: 'overlay-mongo-index-v1', target: 'ls', apply: async () => {} }, + { protocol: 'overlay-mongo-index-v1', target: 'ls', apply: async () => {} } + ] + }) + ).toThrow('Duplicate enlisted Mongo lookup index target') + }) + + test('rejects digest mismatch and unsupported projectors before the body', async () => { + const adapter = storage() + const mismatched = plan() + mismatched.key.semanticDigest = '00'.repeat(32) + expect(await adapter.commitAdmission(mismatched)).toEqual({ + state: 'rejected', + code: 'digest-mismatch' + }) + const invalid = plan() + invalid.identity.txid = 'not-a-hash' + expect(await adapter.commitAdmission(invalid)).toEqual({ + state: 'rejected', + code: 'digest-mismatch' + }) + expect( + await storage({ projector: { protocol: 'not-replay-safe' } }).commitAdmission(plan()) + ).toEqual({ state: 'rejected', code: 'unsupported-projection' }) + const historical = plan() + historical.identity.mode = 'historical' + historical.key.semanticDigest = admissionSemanticDigest(historical.identity) + expect(await adapter.commitAdmission(historical)).toEqual({ + state: 'rejected', + code: 'invalid-plan' + }) + expect( + await storage({ + enlistedIndexes: [ + { + protocol: 'overlay-mongo-index-v1', + target: 'ls_contract', + apply: async () => {} + } + ] + }).commitAdmission(plan()) + ).toEqual({ state: 'rejected', code: 'invalid-plan' }) + }) + + test('returns an already committed or rejected reconcile without running the body', async () => { + const committed: AdmissionReceipt = receiptFor(plan()) + const run = jest.fn() + expect( + await storage({ + runner: runner({ + reconcile: async () => ({ state: 'committed', receipt: committed }), + run + }) + }).commitAdmission(plan()) + ).toEqual({ state: 'committed', receipt: committed }) + expect( + await storage({ + runner: runner({ + reconcile: async () => ({ state: 'rejected', code: 'spend-conflict' }), + run + }) + }).commitAdmission(plan()) + ).toEqual({ state: 'rejected', code: 'spend-conflict' }) + expect(run).not.toHaveBeenCalled() + }) + + test('retries Mongo write conflicts then commits', async () => { + const value = plan() + const committed = receiptFor(value) + const run = jest + .fn() + .mockRejectedValueOnce({ code: 112, message: 'transient' }) + .mockRejectedValueOnce({ code: '112' }) + .mockRejectedValueOnce({ codeName: 'WriteConflict' }) + .mockRejectedValueOnce(new Error('Write conflict')) + .mockRejectedValueOnce({ errmsg: 'Write conflict' }) + .mockResolvedValueOnce({ state: 'committed', receipt: committed }) + expect(await storage({ runner: runner({ run }) }).commitAdmission(value)).toEqual({ + state: 'committed', + receipt: committed + }) + expect(run).toHaveBeenCalledTimes(6) + }) + + test('maps typed body failures and rethrows unknown errors', async () => { + expect( + await storage({ + runner: runner({ + run: async () => { + throw new AdmissionRejectedError('spend-conflict') + } + }) + }).commitAdmission(plan()) + ).toEqual({ state: 'rejected', code: 'spend-conflict' }) + expect( + await storage({ + runner: runner({ + run: async () => { + throw new MongoReadGuardConflictError({ + scope: referenceScope, + key: 'selection', + expectedVersion: '1' + }) + } + }) + }).commitAdmission(plan()) + ).toEqual({ state: 'rejected', code: 'read-conflict' }) + expect( + await storage({ + runner: runner({ + run: async () => { + throw new Error('Mongo payload is not ready for reference') + } + }) + }).commitAdmission(plan()) + ).toEqual({ state: 'rejected', code: 'payload-not-ready' }) + await expect( + storage({ + runner: runner({ + run: async () => { + throw new Error('disk full') + } + }) + }).commitAdmission(plan()) + ).rejects.toThrow('disk full') + }) + + test('waits for a pending attempt, including abort then a fresh body', async () => { + const committed = receiptFor(plan()) + const reconcile = jest + .fn() + .mockResolvedValueOnce({ state: 'pending', attemptId: 'att-1' }) + .mockResolvedValueOnce({ state: 'committed', receipt: committed }) + expect(await storage({ runner: runner({ reconcile }) }).commitAdmission(plan())).toEqual({ + state: 'committed', + receipt: committed + }) + + const run = jest.fn(async () => ({ state: 'committed' as const, receipt: committed })) + const aborted = jest + .fn() + .mockResolvedValueOnce({ state: 'pending', attemptId: 'att-2' }) + .mockResolvedValueOnce({ state: 'aborted' }) + expect( + await storage({ runner: runner({ reconcile: aborted, run }) }).commitAdmission(plan()) + ).toEqual({ state: 'committed', receipt: committed }) + expect(run).toHaveBeenCalledTimes(1) + + const fromRun = jest + .fn() + .mockResolvedValueOnce({ state: 'pending', attemptId: 'unlocated' }) + .mockResolvedValueOnce({ state: 'committed', receipt: committed }) + expect( + await storage({ + runner: runner({ + reconcile: fromRun, + run: async () => ({ state: 'pending', attemptId: 'body-1' }) + }) + }).commitAdmission(plan()) + ).toEqual({ state: 'committed', receipt: committed }) + }) + + test('publishes history-update payloads before the write-conflict retry loop', async () => { + const value = plan() + value.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '4', + affectedFromHeight: '99', + handoff: { + expected: { + scope: referenceScope, + topic: 'tm_contract', + peerId: 'peer-a', + jobId: 'repair-a', + leaseToken: '9', + expiresAtMs: '100', + chainEpoch: '7', + topicHistoryGeneration: '3' + }, + checkpoint: 'repair-checkpoint' + } + } + const publish = jest.fn(async () => ({ + kind: 'outbox-data' as const, + digest: 'aa'.repeat(32), + byteLength: '1' + })) + const committed = receiptFor(value) + expect( + await storage({ + payloads: { publish } as never, + runner: runner({ + run: async () => ({ state: 'committed', receipt: committed }) + }) + }).commitAdmission(value) + ).toEqual({ state: 'committed', receipt: committed }) + expect(publish).toHaveBeenCalledTimes(1) + }) + + test('rejects an out-of-range outbox lease before touching storage', async () => { + const adapter = storage() + await expect(adapter.claimOutbox('lookup', 0)).rejects.toThrow('Invalid Mongo outbox lease') + await expect(adapter.claimOutbox('lookup', 60_001)).rejects.toThrow( + 'Invalid Mongo outbox lease' + ) + await expect(adapter.claimOutbox('propagation', 1.5)).rejects.toThrow( + 'Invalid Mongo outbox lease' + ) + }) +}) + +describe('Mongo overlay storage admission guards', () => { + test('updateTransactionBEEF is intentionally unimplemented', async () => { + const overlay = new MongoOverlayStorage(dummyDb, referenceScope) + const txid = 'ab'.repeat(32) + await expect(overlay.updateTransactionBEEF(txid, [])).rejects.toThrow( + `Mongo overlay storage does not implement updateTransactionBEEF for ${txid}` + ) + expect(overlay.enlistedIndexTargets()).toEqual([]) + await expect( + overlay.publishAdmissionPayload({ + kind: 'not-a-kind' as 'locking-script', + bytes: new Uint8Array([1]) + }) + ).rejects.toThrow('Invalid Mongo admission payload kind') + expect(asStorageUint64('0')).toBe('0') + }) +}) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts index 936f9695d..654390d0e 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts @@ -3,7 +3,14 @@ import { admissionPlan, admissionStorageContract } from '../admission/AdmissionS import { getAdmissionStorage } from '../../storage/AdmissionStorage.js' import { MongoAdmissionStorage } from '../../storage/mongo/MongoAdmissionStorage.js' import { MongoOverlayStorage } from '../../storage/mongo/MongoOverlayStorage.js' -import { bootstrapMongoOverlay, MongoCollectionNames } from '../../storage/mongo/MongoSchema.js' +import { + bootstrapMongoOverlay, + encodeMongoOutputIndex, + encodeMongoUint64, + MongoCollectionNames, + mongoNodeKey, + mongoRecordKey +} from '../../storage/mongo/MongoSchema.js' import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' import { MongoAdmissionHarness } from './MongoAdmissionHarness.js' import type { MongoEnlistedLookupIndex } from '../../storage/mongo/MongoAdmissionStorage.js' @@ -327,6 +334,223 @@ describe('Mongo admission storage', () => { expect(await replacement.commitAdmission(plan)).toEqual(committed) await replacement.close() }, 60000) + + test('covers overlay storage fences, CRUD filters, and unimplemented BEEF updates', async () => { + await harness.reset() + const storage = new MongoOverlayStorage(fixture.db, fixture.scope) + expect(await storage.getHistoryFence('tm_missing')).toEqual({ + chainEpoch: '0', + topicHistoryGeneration: '0' + }) + const now = new Date() + await storage.admission.generations().insertOne({ + _id: storage.admission.generationId('tm_contract'), + schemaVersion: 1, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash, + nodeId: fixture.scope.nodeId, + topic: 'tm_contract', + chainEpoch: encodeMongoUint64('3'), + topicHistoryGeneration: encodeMongoUint64('4'), + policyId: 'policy-1', + createdAt: now, + updatedAt: now + }) + expect(await storage.getHistoryFence('tm_contract')).toEqual({ + chainEpoch: '3', + topicHistoryGeneration: '4' + }) + + const published = await storage.publishAdmissionPayload({ + kind: 'locking-script', + bytes: new Uint8Array([0x76, 0xa9]) + }) + expect(published.byteLength).toBe('2') + expect(published.digest).toHaveLength(64) + + const beef = hydrationBeef() + const txid = Transaction.fromBEEF(beef).id('hex') + const script = [0x76, 0xa9, 0x14, 0x00, 0x88, 0xac] + await storage.insertOutput({ + txid, + outputIndex: 0, + outputScript: script, + satoshis: 50, + topic: 'tm_contract', + spent: false, + outputsConsumed: [], + consumedBy: [], + score: 8, + beef + }) + expect(await storage.findOutput(txid, 0, undefined, false)).toEqual( + expect.objectContaining({ txid, spent: false, satoshis: 50 }) + ) + expect(await storage.findOutput(txid, 0, 'tm_contract', true)).toBeNull() + expect(await storage.findOutputsForTransaction(txid)).toHaveLength(1) + expect(await storage.findUTXOsForTopic('tm_contract', 0, 0)).toHaveLength(1) + expect(await storage.findUTXOsForTopic('tm_contract', 8, 1, true)).toHaveLength(1) + expect(await storage.findUTXOsForTopic('tm_contract', 9, 1)).toHaveLength(0) + + await storage.updateConsumedBy(txid, 0, 'tm_contract', [ + { txid: 'ab'.repeat(32), outputIndex: 1 } + ]) + await storage.insertAppliedTransaction({ txid, topic: 'tm_contract', proven: false }) + await storage.insertAppliedTransaction({ + txid: 'dd'.repeat(32), + topic: 'tm_contract', + proven: true + }) + expect(await storage.doesAppliedTransactionExist({ txid, topic: 'tm_contract' })).toBe(true) + expect( + await storage.doesAppliedTransactionExist({ txid: 'cd'.repeat(32), topic: 'tm_contract' }) + ).toBe(false) + await storage.updateLastInteraction('peer.example', 'tm_contract', 42) + expect(await storage.getLastInteraction('peer.example', 'tm_contract')).toBe(42) + expect(await storage.getLastInteraction('missing', 'tm_contract')).toBe(0) + + await storage.markUTXOAsSpent(txid, 0, 'tm_contract') + expect(await storage.findOutput(txid, 0, 'tm_contract', true)).toEqual( + expect.objectContaining({ spent: true }) + ) + await storage.deleteOutput(txid, 0, 'tm_contract') + expect(await storage.findOutput(txid, 0, 'tm_contract', false)).toBeNull() + await expect(storage.updateTransactionBEEF(txid, [])).rejects.toThrow( + `Mongo overlay storage does not implement updateTransactionBEEF for ${txid}` + ) + + const outputDoc = { + schemaVersion: 1, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash, + nodeId: fixture.scope.nodeId, + scriptPayloadId: storage.admission.payloadId(published), + scriptOffset: encodeMongoUint64('0'), + scriptByteLength: encodeMongoUint64(published.byteLength), + state: 'unspent' as const, + version: '1', + createdAt: now, + updatedAt: now + } + await fixture.db.collection(MongoCollectionNames.outputs).insertOne({ + ...outputDoc, + _id: mongoRecordKey( + mongoNodeKey(fixture.scope), + 'output', + 'tm_unsafe', + 'ee'.repeat(32), + encodeMongoOutputIndex('0') + ), + topic: 'tm_unsafe', + txid: 'ee'.repeat(32), + outputIndex: encodeMongoOutputIndex('0'), + satoshis: encodeMongoUint64('9007199254740993'), + score: encodeMongoUint64('1') + }) + await expect(storage.findOutput('ee'.repeat(32), 0, 'tm_unsafe')).rejects.toThrow( + 'Mongo satoshis exceeds a safe JavaScript integer' + ) + + await fixture.db.collection(MongoCollectionNames.outputs).insertOne({ + ...outputDoc, + _id: mongoRecordKey( + mongoNodeKey(fixture.scope), + 'output', + 'tm_score', + 'cc'.repeat(32), + encodeMongoOutputIndex('0') + ), + topic: 'tm_score', + txid: 'cc'.repeat(32), + outputIndex: encodeMongoOutputIndex('0'), + satoshis: encodeMongoUint64('1'), + score: encodeMongoUint64('9007199254740993') + }) + await expect(storage.findOutput('cc'.repeat(32), 0, 'tm_score')).rejects.toThrow( + 'Mongo score exceeds a safe JavaScript integer' + ) + + await fixture.db.collection(MongoCollectionNames.outputs).insertOne({ + ...outputDoc, + _id: mongoRecordKey( + mongoNodeKey(fixture.scope), + 'output', + 'tm_script', + 'ff'.repeat(32), + encodeMongoOutputIndex('0') + ), + topic: 'tm_script', + txid: 'ff'.repeat(32), + outputIndex: encodeMongoOutputIndex('0'), + satoshis: encodeMongoUint64('1'), + score: encodeMongoUint64('1'), + scriptPayloadId: 'missing-payload' + }) + await expect(storage.findOutput('ff'.repeat(32), 0, 'tm_script')).rejects.toThrow( + 'Mongo output script payload is not ready' + ) + await expect(harness.adapter.acknowledgeOutbox('lookup', 'missing-event')).rejects.toThrow( + 'Mongo outbox event is not leased' + ) + await storage.close() + }) + + test('rejects a missing eviction and a history update from an empty fence', async () => { + await harness.reset() + const missingEviction = admissionPlan('missing-eviction') + missingEviction.decisions[0].evictions = [{ txid: 'aa'.repeat(32), outputIndex: '0' }] + for (const decision of missingEviction.decisions) { + await harness.seed.history( + missingEviction.identity.scope, + decision.topic, + decision.expectedHistory + ) + for (const read of decision.reads) { + if (read.expectedVersion !== null) { + await harness.seed.read( + missingEviction.identity.scope, + decision.topic, + read.key, + read.expectedVersion + ) + } + } + for (const spend of decision.spends) { + await harness.seed.spendable( + missingEviction.identity.scope, + decision.topic, + spend.outpoint, + spend.expectedVersion + ) + } + } + for (const payload of missingEviction.payloads) await harness.seed.readyPayload(payload) + expect(await harness.adapter.commitAdmission(missingEviction)).toEqual({ + state: 'rejected', + code: 'invalid-plan' + }) + + await harness.reset() + const fromZero = admissionPlan('history-zero') + fromZero.decisions[0].expectedHistory = { chainEpoch: '0', topicHistoryGeneration: '0' } + fromZero.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '1', + affectedFromHeight: '0' + } + fromZero.decisions[0].spends = [] + for (const read of fromZero.decisions[0].reads) { + if (read.expectedVersion !== null) { + await harness.seed.read( + fromZero.identity.scope, + fromZero.decisions[0].topic, + read.key, + read.expectedVersion + ) + } + } + for (const payload of fromZero.payloads) await harness.seed.readyPayload(payload) + expect((await harness.adapter.commitAdmission(fromZero)).state).toBe('committed') + }) }) function hydrationBeef(): number[] { From 3e59bb8efaae6e73cfb79889e4bf10fb07a448a3 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 01:37:55 -0500 Subject: [PATCH 091/168] fix(sonar): use toHaveLength in Engine admission tests --- packages/overlays/overlay/src/__tests/Engine.admission.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/overlays/overlay/src/__tests/Engine.admission.test.ts b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts index 6952a3460..0c6229082 100644 --- a/packages/overlays/overlay/src/__tests/Engine.admission.test.ts +++ b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts @@ -304,7 +304,7 @@ describe('Engine admission submit', () => { }) expect(submitted.decisions[0].spends).toHaveLength(1) expect(submitted.outbox.some(intent => intent.target === 'Hello')).toBe(false) - expect(submitted.identity.contextDigest.length).toBe(64) + expect(submitted.identity.contextDigest).toHaveLength(64) }) test('surfaces admission rejection and recovers a pending receipt', async () => { From bb3614e44a2acaa9e444660ba56b95c0b8ba51d9 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:44:18 -0500 Subject: [PATCH 092/168] fix(overlay): reject unsafe Mongo GASP cursor values --- .../__tests/mongo/MongoAdmissionRetry.test.ts | 37 +++++++++++++++++++ .../src/storage/mongo/MongoOverlayStorage.ts | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionRetry.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionRetry.test.ts index 16d6656c9..b47c50ee9 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionRetry.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionRetry.test.ts @@ -13,11 +13,20 @@ import { import { MongoAdmissionStorage } from '../../storage/mongo/MongoAdmissionStorage.js' import { MongoOverlayStorage } from '../../storage/mongo/MongoOverlayStorage.js' import type { MongoTransactionRunner } from '../../storage/mongo/MongoTransactionRunner.js' +import { encodeMongoUint64 } from '../../storage/mongo/MongoSchema.js' import { admissionPlan } from '../admission/AdmissionStorageContract.js' import { referenceScope } from '../admission/ReferenceAdmissionStorage.js' const dummyDb = { collection: () => ({}) } as unknown as Db +function cursorDb(score?: string): Db { + return { + collection: () => ({ + findOne: async () => (score === undefined ? null : { score }) + }) + } as unknown as Db +} + const clone = (value: T): T => structuredClone(value) function plan(): AdmissionCommit { @@ -287,6 +296,34 @@ describe('Mongo admission commit guards and write-conflict retry', () => { }) describe('Mongo overlay storage admission guards', () => { + test('preserves safe GASP cursors and rejects unsafe persisted values', async () => { + const maxSafe = BigInt(Number.MAX_SAFE_INTEGER) + const getCursor = (score?: string): Promise => + new MongoOverlayStorage(cursorDb(score), referenceScope).getLastInteraction( + 'peer.example', + 'tm_contract' + ) + + await expect(getCursor(encodeMongoUint64('42'))).resolves.toBe(42) + await expect(getCursor()).resolves.toBe(0) + await expect(getCursor(encodeMongoUint64(maxSafe.toString()))).resolves.toBe( + Number.MAX_SAFE_INTEGER + ) + await expect(getCursor(encodeMongoUint64((maxSafe + 1n).toString()))).rejects.toThrow( + 'Mongo GASP cursor exceeds a safe JavaScript integer' + ) + await expect(getCursor(encodeMongoUint64((maxSafe + 3n).toString()))).rejects.toThrow( + 'Mongo GASP cursor exceeds a safe JavaScript integer' + ) + // These specific values rounded down/up in the old Number-only decoder. + await expect(getCursor(encodeMongoUint64('9007199254740993'))).rejects.toThrow( + 'Mongo GASP cursor exceeds a safe JavaScript integer' + ) + await expect(getCursor(encodeMongoUint64('9007199254740995'))).rejects.toThrow( + 'Mongo GASP cursor exceeds a safe JavaScript integer' + ) + }) + test('updateTransactionBEEF is intentionally unimplemented', async () => { const overlay = new MongoOverlayStorage(dummyDb, referenceScope) const txid = 'ab'.repeat(32) diff --git a/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts index cd69c2cd5..4dcae8744 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts @@ -372,7 +372,7 @@ export class MongoOverlayStorage implements Storage { topic }) if (found?.score === undefined) return 0 - return Number(decodeMongoUint64(found.score as string)) + return this.toSafeNumber(decodeMongoUint64(found.score as string), 'GASP cursor') } private outputId(topic: string, txid: string, outputIndex: number): string { From 8f8ddb7083d6042487936f71b70bbc633f36c475 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:41:38 -0500 Subject: [PATCH 093/168] fix(ci): parent and finalize Codecov coverage reports --- .github/workflows/ci.yml | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c55dbc872..354df01fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1115,11 +1115,43 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ${{ steps.cov.outputs.files }} - name: ts-stack + # Make the current attempt distinguishable from a previous rerun of + # the same commit when waiting for Codecov to finish processing it. + name: ts-stack-${{ github.run_id }}-${{ github.run_attempt }} slug: bsv-blockchain/ts-stack + # CI checks out GitHub's synthetic PR merge commit but Codecov + # reports against the branch head. Give Codecov the actual base + # parent so it does not fall back to its last covered main commit. + commit_parent: ${{ github.event.pull_request.base.sha || github.event.before }} # Coverage upload is non-blocking: Codecov CLI GPG-verify/outages must not gate CI. fail_ci_if_error: false + - name: Wait for Codecov to merge the uploaded report + if: steps.cov.outputs.has-coverage == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + env: + CODECOV_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + CODECOV_BUILD_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + CODECOV_UPLOAD_NAME: ts-stack-${{ github.run_id }}-${{ github.run_attempt }} + run: | + deadline=$((SECONDS + 300)) + commit_url="https://api.codecov.io/api/v2/github/bsv-blockchain/repos/ts-stack/commits/${CODECOV_COMMIT}/" + uploads_url="${commit_url}uploads/" + while true; do + if commit=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$commit_url") && + [ "$(jq --raw-output '.state // empty' <<<"$commit")" = 'complete' ] && + uploads=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$uploads_url") && + [ "$(jq --arg build_url "$CODECOV_BUILD_URL" --arg name "$CODECOV_UPLOAD_NAME" '[.results[]? | select(.build_url == $build_url and .name == $name and .state_name == "MERGED")] | length' <<<"$uploads")" -gt 0 ]; then + echo "Codecov merged ${CODECOV_UPLOAD_NAME} for ${CODECOV_COMMIT}." + break + fi + + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::Codecov did not merge coverage for ${CODECOV_COMMIT} within five minutes." + exit 1 + fi + sleep 5 + done + - name: Publish finalized Codecov notifications if: steps.cov.outputs.has-coverage == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 From 6ab9be46a452170ac85c5dddea355ef8bfd6f521 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:41:38 -0500 Subject: [PATCH 094/168] fix(ci): parent and finalize Codecov coverage reports --- .github/workflows/ci.yml | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c55dbc872..354df01fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1115,11 +1115,43 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ${{ steps.cov.outputs.files }} - name: ts-stack + # Make the current attempt distinguishable from a previous rerun of + # the same commit when waiting for Codecov to finish processing it. + name: ts-stack-${{ github.run_id }}-${{ github.run_attempt }} slug: bsv-blockchain/ts-stack + # CI checks out GitHub's synthetic PR merge commit but Codecov + # reports against the branch head. Give Codecov the actual base + # parent so it does not fall back to its last covered main commit. + commit_parent: ${{ github.event.pull_request.base.sha || github.event.before }} # Coverage upload is non-blocking: Codecov CLI GPG-verify/outages must not gate CI. fail_ci_if_error: false + - name: Wait for Codecov to merge the uploaded report + if: steps.cov.outputs.has-coverage == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + env: + CODECOV_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + CODECOV_BUILD_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + CODECOV_UPLOAD_NAME: ts-stack-${{ github.run_id }}-${{ github.run_attempt }} + run: | + deadline=$((SECONDS + 300)) + commit_url="https://api.codecov.io/api/v2/github/bsv-blockchain/repos/ts-stack/commits/${CODECOV_COMMIT}/" + uploads_url="${commit_url}uploads/" + while true; do + if commit=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$commit_url") && + [ "$(jq --raw-output '.state // empty' <<<"$commit")" = 'complete' ] && + uploads=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$uploads_url") && + [ "$(jq --arg build_url "$CODECOV_BUILD_URL" --arg name "$CODECOV_UPLOAD_NAME" '[.results[]? | select(.build_url == $build_url and .name == $name and .state_name == "MERGED")] | length' <<<"$uploads")" -gt 0 ]; then + echo "Codecov merged ${CODECOV_UPLOAD_NAME} for ${CODECOV_COMMIT}." + break + fi + + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::Codecov did not merge coverage for ${CODECOV_COMMIT} within five minutes." + exit 1 + fi + sleep 5 + done + - name: Publish finalized Codecov notifications if: steps.cov.outputs.has-coverage == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 From 53a176575280ad33515d16c191402e25c25f684e Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:41:38 -0500 Subject: [PATCH 095/168] fix(ci): parent and finalize Codecov coverage reports --- .github/workflows/ci.yml | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c55dbc872..354df01fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1115,11 +1115,43 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ${{ steps.cov.outputs.files }} - name: ts-stack + # Make the current attempt distinguishable from a previous rerun of + # the same commit when waiting for Codecov to finish processing it. + name: ts-stack-${{ github.run_id }}-${{ github.run_attempt }} slug: bsv-blockchain/ts-stack + # CI checks out GitHub's synthetic PR merge commit but Codecov + # reports against the branch head. Give Codecov the actual base + # parent so it does not fall back to its last covered main commit. + commit_parent: ${{ github.event.pull_request.base.sha || github.event.before }} # Coverage upload is non-blocking: Codecov CLI GPG-verify/outages must not gate CI. fail_ci_if_error: false + - name: Wait for Codecov to merge the uploaded report + if: steps.cov.outputs.has-coverage == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + env: + CODECOV_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + CODECOV_BUILD_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + CODECOV_UPLOAD_NAME: ts-stack-${{ github.run_id }}-${{ github.run_attempt }} + run: | + deadline=$((SECONDS + 300)) + commit_url="https://api.codecov.io/api/v2/github/bsv-blockchain/repos/ts-stack/commits/${CODECOV_COMMIT}/" + uploads_url="${commit_url}uploads/" + while true; do + if commit=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$commit_url") && + [ "$(jq --raw-output '.state // empty' <<<"$commit")" = 'complete' ] && + uploads=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$uploads_url") && + [ "$(jq --arg build_url "$CODECOV_BUILD_URL" --arg name "$CODECOV_UPLOAD_NAME" '[.results[]? | select(.build_url == $build_url and .name == $name and .state_name == "MERGED")] | length' <<<"$uploads")" -gt 0 ]; then + echo "Codecov merged ${CODECOV_UPLOAD_NAME} for ${CODECOV_COMMIT}." + break + fi + + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::Codecov did not merge coverage for ${CODECOV_COMMIT} within five minutes." + exit 1 + fi + sleep 5 + done + - name: Publish finalized Codecov notifications if: steps.cov.outputs.has-coverage == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 From 0217e99716d849afc81edcd73da10990e57dbf89 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:41:38 -0500 Subject: [PATCH 096/168] fix(ci): parent and finalize Codecov coverage reports --- .github/workflows/ci.yml | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c55dbc872..354df01fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1115,11 +1115,43 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ${{ steps.cov.outputs.files }} - name: ts-stack + # Make the current attempt distinguishable from a previous rerun of + # the same commit when waiting for Codecov to finish processing it. + name: ts-stack-${{ github.run_id }}-${{ github.run_attempt }} slug: bsv-blockchain/ts-stack + # CI checks out GitHub's synthetic PR merge commit but Codecov + # reports against the branch head. Give Codecov the actual base + # parent so it does not fall back to its last covered main commit. + commit_parent: ${{ github.event.pull_request.base.sha || github.event.before }} # Coverage upload is non-blocking: Codecov CLI GPG-verify/outages must not gate CI. fail_ci_if_error: false + - name: Wait for Codecov to merge the uploaded report + if: steps.cov.outputs.has-coverage == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + env: + CODECOV_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + CODECOV_BUILD_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + CODECOV_UPLOAD_NAME: ts-stack-${{ github.run_id }}-${{ github.run_attempt }} + run: | + deadline=$((SECONDS + 300)) + commit_url="https://api.codecov.io/api/v2/github/bsv-blockchain/repos/ts-stack/commits/${CODECOV_COMMIT}/" + uploads_url="${commit_url}uploads/" + while true; do + if commit=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$commit_url") && + [ "$(jq --raw-output '.state // empty' <<<"$commit")" = 'complete' ] && + uploads=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$uploads_url") && + [ "$(jq --arg build_url "$CODECOV_BUILD_URL" --arg name "$CODECOV_UPLOAD_NAME" '[.results[]? | select(.build_url == $build_url and .name == $name and .state_name == "MERGED")] | length' <<<"$uploads")" -gt 0 ]; then + echo "Codecov merged ${CODECOV_UPLOAD_NAME} for ${CODECOV_COMMIT}." + break + fi + + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::Codecov did not merge coverage for ${CODECOV_COMMIT} within five minutes." + exit 1 + fi + sleep 5 + done + - name: Publish finalized Codecov notifications if: steps.cov.outputs.has-coverage == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 From 644acccb0ff9e8635b30ec602458fced30c5ec2b Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:54:54 -0500 Subject: [PATCH 097/168] docs: reverify resource and TTN rollout gates --- docs/reference/service-resource-profiles.md | 2 +- docs/reference/ttn-rollout.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/service-resource-profiles.md b/docs/reference/service-resource-profiles.md index 64730ffd8..17161602b 100644 --- a/docs/reference/service-resource-profiles.md +++ b/docs/reference/service-resource-profiles.md @@ -4,7 +4,7 @@ title: 'Service Resource Profiles, Scaling, and Message Box Economics' kind: reference version: '1.0.0' last_updated: '2026-08-10' -last_verified: '2026-08-10' +last_verified: '2026-09-09' review_cadence_days: 30 status: stable tags: [reference, infrastructure, resource-safety, scaling, message-box, brc-105] diff --git a/docs/reference/ttn-rollout.md b/docs/reference/ttn-rollout.md index 40390acc8..10f2820d7 100644 --- a/docs/reference/ttn-rollout.md +++ b/docs/reference/ttn-rollout.md @@ -4,7 +4,7 @@ title: 'TerraTestNet Rollout Gate' kind: reference version: '1.0.0' last_updated: '2026-08-10' -last_verified: '2026-08-10' +last_verified: '2026-09-09' review_cadence_days: 30 status: stable tags: [reference, ttn, releases, containers, operations] From 06e7e6b476ab14993c582c24be2dbb86cc2f278c Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:54:54 -0500 Subject: [PATCH 098/168] docs: reverify resource and TTN rollout gates --- docs/reference/service-resource-profiles.md | 2 +- docs/reference/ttn-rollout.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/service-resource-profiles.md b/docs/reference/service-resource-profiles.md index 64730ffd8..17161602b 100644 --- a/docs/reference/service-resource-profiles.md +++ b/docs/reference/service-resource-profiles.md @@ -4,7 +4,7 @@ title: 'Service Resource Profiles, Scaling, and Message Box Economics' kind: reference version: '1.0.0' last_updated: '2026-08-10' -last_verified: '2026-08-10' +last_verified: '2026-09-09' review_cadence_days: 30 status: stable tags: [reference, infrastructure, resource-safety, scaling, message-box, brc-105] diff --git a/docs/reference/ttn-rollout.md b/docs/reference/ttn-rollout.md index 40390acc8..10f2820d7 100644 --- a/docs/reference/ttn-rollout.md +++ b/docs/reference/ttn-rollout.md @@ -4,7 +4,7 @@ title: 'TerraTestNet Rollout Gate' kind: reference version: '1.0.0' last_updated: '2026-08-10' -last_verified: '2026-08-10' +last_verified: '2026-09-09' review_cadence_days: 30 status: stable tags: [reference, ttn, releases, containers, operations] From 89d8670d06d9a3bbd20679ef0b5263ab22f1909b Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:54:54 -0500 Subject: [PATCH 099/168] docs: reverify resource and TTN rollout gates --- docs/reference/service-resource-profiles.md | 2 +- docs/reference/ttn-rollout.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/service-resource-profiles.md b/docs/reference/service-resource-profiles.md index 64730ffd8..17161602b 100644 --- a/docs/reference/service-resource-profiles.md +++ b/docs/reference/service-resource-profiles.md @@ -4,7 +4,7 @@ title: 'Service Resource Profiles, Scaling, and Message Box Economics' kind: reference version: '1.0.0' last_updated: '2026-08-10' -last_verified: '2026-08-10' +last_verified: '2026-09-09' review_cadence_days: 30 status: stable tags: [reference, infrastructure, resource-safety, scaling, message-box, brc-105] diff --git a/docs/reference/ttn-rollout.md b/docs/reference/ttn-rollout.md index 40390acc8..10f2820d7 100644 --- a/docs/reference/ttn-rollout.md +++ b/docs/reference/ttn-rollout.md @@ -4,7 +4,7 @@ title: 'TerraTestNet Rollout Gate' kind: reference version: '1.0.0' last_updated: '2026-08-10' -last_verified: '2026-08-10' +last_verified: '2026-09-09' review_cadence_days: 30 status: stable tags: [reference, ttn, releases, containers, operations] From 61d44463a66ea0c43f11b850cbf17bcf3f830816 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:54:54 -0500 Subject: [PATCH 100/168] docs: reverify resource and TTN rollout gates --- docs/reference/service-resource-profiles.md | 2 +- docs/reference/ttn-rollout.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/service-resource-profiles.md b/docs/reference/service-resource-profiles.md index 64730ffd8..17161602b 100644 --- a/docs/reference/service-resource-profiles.md +++ b/docs/reference/service-resource-profiles.md @@ -4,7 +4,7 @@ title: 'Service Resource Profiles, Scaling, and Message Box Economics' kind: reference version: '1.0.0' last_updated: '2026-08-10' -last_verified: '2026-08-10' +last_verified: '2026-09-09' review_cadence_days: 30 status: stable tags: [reference, infrastructure, resource-safety, scaling, message-box, brc-105] diff --git a/docs/reference/ttn-rollout.md b/docs/reference/ttn-rollout.md index 40390acc8..10f2820d7 100644 --- a/docs/reference/ttn-rollout.md +++ b/docs/reference/ttn-rollout.md @@ -4,7 +4,7 @@ title: 'TerraTestNet Rollout Gate' kind: reference version: '1.0.0' last_updated: '2026-08-10' -last_verified: '2026-08-10' +last_verified: '2026-09-09' review_cadence_days: 30 status: stable tags: [reference, ttn, releases, containers, operations] From b65c3c652fc1e935f2c4d67d02c1f780b81837b9 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:59:53 -0500 Subject: [PATCH 101/168] fix(ci): require HTTPS for Codecov polling --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 354df01fc..8aa79fb5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1137,9 +1137,9 @@ jobs: commit_url="https://api.codecov.io/api/v2/github/bsv-blockchain/repos/ts-stack/commits/${CODECOV_COMMIT}/" uploads_url="${commit_url}uploads/" while true; do - if commit=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$commit_url") && + if commit=$(curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --connect-timeout 5 --max-time 15 "$commit_url") && [ "$(jq --raw-output '.state // empty' <<<"$commit")" = 'complete' ] && - uploads=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$uploads_url") && + uploads=$(curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --connect-timeout 5 --max-time 15 "$uploads_url") && [ "$(jq --arg build_url "$CODECOV_BUILD_URL" --arg name "$CODECOV_UPLOAD_NAME" '[.results[]? | select(.build_url == $build_url and .name == $name and .state_name == "MERGED")] | length' <<<"$uploads")" -gt 0 ]; then echo "Codecov merged ${CODECOV_UPLOAD_NAME} for ${CODECOV_COMMIT}." break From e966d7afbc1699e317414cfc2396186ad0c69522 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:59:53 -0500 Subject: [PATCH 102/168] fix(ci): require HTTPS for Codecov polling --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 354df01fc..8aa79fb5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1137,9 +1137,9 @@ jobs: commit_url="https://api.codecov.io/api/v2/github/bsv-blockchain/repos/ts-stack/commits/${CODECOV_COMMIT}/" uploads_url="${commit_url}uploads/" while true; do - if commit=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$commit_url") && + if commit=$(curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --connect-timeout 5 --max-time 15 "$commit_url") && [ "$(jq --raw-output '.state // empty' <<<"$commit")" = 'complete' ] && - uploads=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$uploads_url") && + uploads=$(curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --connect-timeout 5 --max-time 15 "$uploads_url") && [ "$(jq --arg build_url "$CODECOV_BUILD_URL" --arg name "$CODECOV_UPLOAD_NAME" '[.results[]? | select(.build_url == $build_url and .name == $name and .state_name == "MERGED")] | length' <<<"$uploads")" -gt 0 ]; then echo "Codecov merged ${CODECOV_UPLOAD_NAME} for ${CODECOV_COMMIT}." break From 743e29fdde50b00798460ee8cb58273a125f4c96 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:59:53 -0500 Subject: [PATCH 103/168] fix(ci): require HTTPS for Codecov polling --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 354df01fc..8aa79fb5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1137,9 +1137,9 @@ jobs: commit_url="https://api.codecov.io/api/v2/github/bsv-blockchain/repos/ts-stack/commits/${CODECOV_COMMIT}/" uploads_url="${commit_url}uploads/" while true; do - if commit=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$commit_url") && + if commit=$(curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --connect-timeout 5 --max-time 15 "$commit_url") && [ "$(jq --raw-output '.state // empty' <<<"$commit")" = 'complete' ] && - uploads=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$uploads_url") && + uploads=$(curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --connect-timeout 5 --max-time 15 "$uploads_url") && [ "$(jq --arg build_url "$CODECOV_BUILD_URL" --arg name "$CODECOV_UPLOAD_NAME" '[.results[]? | select(.build_url == $build_url and .name == $name and .state_name == "MERGED")] | length' <<<"$uploads")" -gt 0 ]; then echo "Codecov merged ${CODECOV_UPLOAD_NAME} for ${CODECOV_COMMIT}." break From 9916e8966beee339895bd4fbb000c17875617182 Mon Sep 17 00:00:00 2001 From: Deggen Date: Wed, 9 Sep 2026 21:59:53 -0500 Subject: [PATCH 104/168] fix(ci): require HTTPS for Codecov polling --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 354df01fc..8aa79fb5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1137,9 +1137,9 @@ jobs: commit_url="https://api.codecov.io/api/v2/github/bsv-blockchain/repos/ts-stack/commits/${CODECOV_COMMIT}/" uploads_url="${commit_url}uploads/" while true; do - if commit=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$commit_url") && + if commit=$(curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --connect-timeout 5 --max-time 15 "$commit_url") && [ "$(jq --raw-output '.state // empty' <<<"$commit")" = 'complete' ] && - uploads=$(curl --fail --silent --show-error --location --connect-timeout 5 --max-time 15 "$uploads_url") && + uploads=$(curl --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --connect-timeout 5 --max-time 15 "$uploads_url") && [ "$(jq --arg build_url "$CODECOV_BUILD_URL" --arg name "$CODECOV_UPLOAD_NAME" '[.results[]? | select(.build_url == $build_url and .name == $name and .state_name == "MERGED")] | length' <<<"$uploads")" -gt 0 ]; then echo "Codecov merged ${CODECOV_UPLOAD_NAME} for ${CODECOV_COMMIT}." break From bf770cdd3f4bac1ccdfbd327a6beb5a8b1060886 Mon Sep 17 00:00:00 2001 From: Deggen Date: Thu, 10 Sep 2026 14:13:15 -0500 Subject: [PATCH 105/168] fix(ci): raise SDK esbuild raw browser budget --- packages/sdk/browser-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index 6705fcce5..f72f855e8 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -21,7 +21,7 @@ "brotli": 165000 }, "esbuild": { - "raw": 590000, + "raw": 600000, "gzip": 180000, "brotli": 150000 } From f73813430ff84cb437eb9882ca41c82539fee39d Mon Sep 17 00:00:00 2001 From: Deggen Date: Thu, 10 Sep 2026 14:31:53 -0500 Subject: [PATCH 106/168] test(sdk): cover invalid evidence intake limits --- .../__tests/LookupResolver.evidence.test.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.evidence.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.evidence.test.ts index 6c10ebcb6..969f12bf6 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.evidence.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.evidence.test.ts @@ -72,6 +72,28 @@ function resolverFor( } describe('LookupResolver additive evidence intake', () => { + it('rejects unusable evidence limits before starting a lookup', async () => { + const resolver = resolverFor(['https://limits.example'], async () => ({ + type: 'output-list', + outputs: [] + })) + + await expect( + resolver.query( + { service, query: {} }, + undefined, + { evidenceLimits: { maxOutputs: 0, maxBytes: 1 } } + ) + ).rejects.toThrow('Evidence intake limits must be positive safe integers') + await expect( + resolver.query( + { service, query: {} }, + undefined, + { evidenceLimits: { maxOutputs: 1, maxBytes: Number.POSITIVE_INFINITY } } + ) + ).rejects.toThrow('Evidence intake limits must be positive safe integers') + }) + it('delivers both owned receipts before legacy first-wins aggregation and isolates callback mutations', async () => { const { bad, valid } = await sameTransactionReceipts() const firstHost = 'https://first.invalid-proof.example' From 27180ea92ee3b462532940b483b438be5fb9a1f3 Mon Sep 17 00:00:00 2001 From: Deggen Date: Thu, 10 Sep 2026 14:32:14 -0500 Subject: [PATCH 107/168] test(overlay): cover uint64 payload length boundary --- .../overlay/src/__tests/mongo/MongoGuardBranches.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoGuardBranches.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoGuardBranches.test.ts index 6b1a47664..b8e5e8088 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoGuardBranches.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoGuardBranches.test.ts @@ -150,6 +150,14 @@ describe('Mongo payload store input and operation guards', () => { await expect( store.publish({ kind: 'outbox-data', digest, byteLength: '01', bytes: bytes() }) ).rejects.toThrow('Invalid Mongo payload byte length') + await expect( + store.publish({ + kind: 'outbox-data', + digest, + byteLength: '18446744073709551616', + bytes: bytes() + }) + ).rejects.toThrow('Invalid Mongo payload byte length') await expect( store.publish({ kind: 'outbox-data', From a204c5bb545a587a64570a9cfc565cd799634ec4 Mon Sep 17 00:00:00 2001 From: Deggen Date: Thu, 10 Sep 2026 14:32:46 -0500 Subject: [PATCH 108/168] test(overlay): cover admission output-index bound --- .../overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts index 94c08689c..f1cdfc9bb 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts @@ -167,6 +167,10 @@ describe('Mongo admission plan validation', () => { }) test('rejects malformed spends, edges, outputs, and applied history', () => { + const outputIndex = clone(admissionPlan('output-index')) + outputIndex.decisions[0].spends[0].outpoint.outputIndex = '4294967296' + expect(validateAdmissionPlan(outputIndex)).toBe('invalid-plan') + const spendTxid = clone(admissionPlan('spend-txid')) spendTxid.decisions[0].spends[0].outpoint.txid = 'nope' expect(validateAdmissionPlan(spendTxid)).toBe('invalid-plan') From 958f7fe18a353c4133a766f7d8cb96c30550ffb3 Mon Sep 17 00:00:00 2001 From: Deggen Date: Thu, 10 Sep 2026 15:37:46 -0500 Subject: [PATCH 109/168] test(overlay): advertise the Go interop listener as explicit loopback --- packages/overlays/overlay/src/__tests/fixtures/README.md | 3 +++ .../overlay/src/__tests/fixtures/basm-go-read-server.go | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/overlays/overlay/src/__tests/fixtures/README.md b/packages/overlays/overlay/src/__tests/fixtures/README.md index 4ad06bdb3..b5d4fb03e 100644 --- a/packages/overlays/overlay/src/__tests/fixtures/README.md +++ b/packages/overlays/overlay/src/__tests/fixtures/README.md @@ -24,3 +24,6 @@ values as authoritative without that independent check. `basm-go-read-server.go` is a local B01 interop host compiled into a temporary module. It replaces onto the Go overlay-services worktree and must not modify that tree's uncommitted S04 files. + +The temporary interop server binds and advertises only `127.0.0.1`; its HTTP +transport never exposes the fixture on an external network interface. diff --git a/packages/overlays/overlay/src/__tests/fixtures/basm-go-read-server.go b/packages/overlays/overlay/src/__tests/fixtures/basm-go-read-server.go index 63f1e6f7e..57eba7cc3 100644 --- a/packages/overlays/overlay/src/__tests/fixtures/basm-go-read-server.go +++ b/packages/overlays/overlay/src/__tests/fixtures/basm-go-read-server.go @@ -77,7 +77,8 @@ func listen(provider engine.BASMProvider) (string, error) { fmt.Fprintln(os.Stderr, serveErr) } }() - return "http://" + listener.Addr().String(), nil + // Keep the advertised host explicitly loopback, matching the bound listener. + return fmt.Sprintf("http://127.0.0.1:%d", listener.Addr().(*net.TCPAddr).Port), nil } func newReadyService() (*engine.BASMReadService, error) { From d591fce24a0fb10e0e0ad19a9f177590e59502ab Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 13:48:32 -0500 Subject: [PATCH 110/168] fix overlay replay identity and evicted output reads --- pnpm-workspace.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index df07a6a66..8dc8eea40 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -44,10 +44,6 @@ overrides: # exactly. Keep this parent-scoped substitution until upstream publishes a # release that depends on qs 6.15.2 or newer. 'typed-rest-client@2.3.1>qs': 6.15.3 - # Jest's istanbul reporting chain (@istanbuljs/load-nyc-config, ts-jest, - # jsdom, webpack-cli) still pulls js-yaml 3.15.1, affected by unbounded CPU - # use on empty merge sources. Select the first patched 3.x release. - js-yaml@<3.15.2: 3.15.2 patchedDependencies: image-size@1.2.1: patches/image-size@1.2.1.patch auditConfig: From 76f52450de49d663d2f267baa21c182b8168babf Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 13:51:16 -0500 Subject: [PATCH 111/168] fix(ci): remove duplicate workspace override --- pnpm-workspace.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index df07a6a66..8dc8eea40 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -44,10 +44,6 @@ overrides: # exactly. Keep this parent-scoped substitution until upstream publishes a # release that depends on qs 6.15.2 or newer. 'typed-rest-client@2.3.1>qs': 6.15.3 - # Jest's istanbul reporting chain (@istanbuljs/load-nyc-config, ts-jest, - # jsdom, webpack-cli) still pulls js-yaml 3.15.1, affected by unbounded CPU - # use on empty merge sources. Select the first patched 3.x release. - js-yaml@<3.15.2: 3.15.2 patchedDependencies: image-size@1.2.1: patches/image-size@1.2.1.patch auditConfig: From 2108286515f367c2c3704d12da6936fe1e0e1631 Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 14:01:59 -0500 Subject: [PATCH 112/168] fix(ci): synchronize admission lockfile overrides --- pnpm-lock.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3368f9c42..9c1ca9409 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,7 +7,6 @@ settings: overrides: brace-expansion@<5.0.9: 5.0.9 nanoid@<3.3.18: 3.3.18 - js-yaml@<3.15.2: 3.15.2 toml@<4.2.0: 4.2.0 typed-rest-client@2.3.1>qs: 6.15.3 From d6dd0c0676942e726c9ed9ee5391ab8b716e8452 Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 14:05:46 -0500 Subject: [PATCH 113/168] fix: make overlay admission runtime portable --- packages/sdk/src/overlay-tools/SHIPBroadcaster.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts index 7179a1c51..0cbb99b83 100644 --- a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts +++ b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts @@ -89,8 +89,13 @@ export class HTTPSOverlayBroadcastFacilitator implements OverlayBroadcastFacilit httpClient: typeof fetch allowHTTP: boolean - constructor(httpClient = fetch, allowHTTP: boolean = false) { - this.httpClient = httpClient + constructor(httpClient?: typeof fetch, allowHTTP: boolean = false) { + const defaultClient = + typeof globalThis !== 'undefined' && typeof globalThis.fetch === 'function' + ? globalThis.fetch.bind(globalThis) + : undefined + if (!httpClient && !defaultClient) throw new Error('Fetch API is unavailable') + this.httpClient = httpClient ?? defaultClient! this.allowHTTP = allowHTTP } From 6cfc0e6731e3911cf821f58208f1e50971e9efdd Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 14:05:46 -0500 Subject: [PATCH 114/168] fix: make overlay admission runtime portable --- .../overlays/overlay/src/storage/AdmissionStorage.ts | 1 + packages/sdk/src/overlay-tools/SHIPBroadcaster.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/overlays/overlay/src/storage/AdmissionStorage.ts b/packages/overlays/overlay/src/storage/AdmissionStorage.ts index 8e1a0ef02..653e19483 100644 --- a/packages/overlays/overlay/src/storage/AdmissionStorage.ts +++ b/packages/overlays/overlay/src/storage/AdmissionStorage.ts @@ -243,6 +243,7 @@ export function admissionSemanticDigest(identity: AdmissionIdentity): string { identity.scope.genesisHash, identity.scope.nodeId, identity.txid, + identity.mode, identity.contextDigest, String(topics.length), ...topics.flatMap(({ topic, policyId }) => [topic, policyId]) diff --git a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts index 7179a1c51..0cbb99b83 100644 --- a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts +++ b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts @@ -89,8 +89,13 @@ export class HTTPSOverlayBroadcastFacilitator implements OverlayBroadcastFacilit httpClient: typeof fetch allowHTTP: boolean - constructor(httpClient = fetch, allowHTTP: boolean = false) { - this.httpClient = httpClient + constructor(httpClient?: typeof fetch, allowHTTP: boolean = false) { + const defaultClient = + typeof globalThis !== 'undefined' && typeof globalThis.fetch === 'function' + ? globalThis.fetch.bind(globalThis) + : undefined + if (!httpClient && !defaultClient) throw new Error('Fetch API is unavailable') + this.httpClient = httpClient ?? defaultClient! this.allowHTTP = allowHTTP } From e46fefcbda07a464f47de330f02ff1ed9adedea3 Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 14:06:00 -0500 Subject: [PATCH 115/168] fix: avoid eager fetch dependency in broadcaster --- packages/sdk/src/overlay-tools/SHIPBroadcaster.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts index 7179a1c51..0cbb99b83 100644 --- a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts +++ b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts @@ -89,8 +89,13 @@ export class HTTPSOverlayBroadcastFacilitator implements OverlayBroadcastFacilit httpClient: typeof fetch allowHTTP: boolean - constructor(httpClient = fetch, allowHTTP: boolean = false) { - this.httpClient = httpClient + constructor(httpClient?: typeof fetch, allowHTTP: boolean = false) { + const defaultClient = + typeof globalThis !== 'undefined' && typeof globalThis.fetch === 'function' + ? globalThis.fetch.bind(globalThis) + : undefined + if (!httpClient && !defaultClient) throw new Error('Fetch API is unavailable') + this.httpClient = httpClient ?? defaultClient! this.allowHTTP = allowHTTP } From 46fa3cafb6e62a68aeff314a734e6764183e16c2 Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 14:06:00 -0500 Subject: [PATCH 116/168] fix: avoid eager fetch dependency in broadcaster --- packages/sdk/src/overlay-tools/SHIPBroadcaster.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts index 7179a1c51..0cbb99b83 100644 --- a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts +++ b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts @@ -89,8 +89,13 @@ export class HTTPSOverlayBroadcastFacilitator implements OverlayBroadcastFacilit httpClient: typeof fetch allowHTTP: boolean - constructor(httpClient = fetch, allowHTTP: boolean = false) { - this.httpClient = httpClient + constructor(httpClient?: typeof fetch, allowHTTP: boolean = false) { + const defaultClient = + typeof globalThis !== 'undefined' && typeof globalThis.fetch === 'function' + ? globalThis.fetch.bind(globalThis) + : undefined + if (!httpClient && !defaultClient) throw new Error('Fetch API is unavailable') + this.httpClient = httpClient ?? defaultClient! this.allowHTTP = allowHTTP } From 740e36ab73414af2d053e5952447465d432feed1 Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 14:07:09 -0500 Subject: [PATCH 117/168] fix: defer unavailable fetch failure until send --- packages/sdk/src/overlay-tools/SHIPBroadcaster.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts index 0cbb99b83..839dc658d 100644 --- a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts +++ b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts @@ -93,9 +93,10 @@ export class HTTPSOverlayBroadcastFacilitator implements OverlayBroadcastFacilit const defaultClient = typeof globalThis !== 'undefined' && typeof globalThis.fetch === 'function' ? globalThis.fetch.bind(globalThis) - : undefined - if (!httpClient && !defaultClient) throw new Error('Fetch API is unavailable') - this.httpClient = httpClient ?? defaultClient! + : (async () => { + throw new Error('Fetch API is unavailable') + }) as typeof fetch + this.httpClient = httpClient ?? defaultClient this.allowHTTP = allowHTTP } From 7c0542d2036cd866844ed6a30b2930e2b5ce7004 Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 14:07:09 -0500 Subject: [PATCH 118/168] fix: defer unavailable fetch failure until send --- packages/sdk/src/overlay-tools/SHIPBroadcaster.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts index 0cbb99b83..839dc658d 100644 --- a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts +++ b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts @@ -93,9 +93,10 @@ export class HTTPSOverlayBroadcastFacilitator implements OverlayBroadcastFacilit const defaultClient = typeof globalThis !== 'undefined' && typeof globalThis.fetch === 'function' ? globalThis.fetch.bind(globalThis) - : undefined - if (!httpClient && !defaultClient) throw new Error('Fetch API is unavailable') - this.httpClient = httpClient ?? defaultClient! + : (async () => { + throw new Error('Fetch API is unavailable') + }) as typeof fetch + this.httpClient = httpClient ?? defaultClient this.allowHTTP = allowHTTP } From 73687697046c3206594016d21da0d615b5f8acb9 Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 14:07:09 -0500 Subject: [PATCH 119/168] fix: defer unavailable fetch failure until send --- packages/sdk/src/overlay-tools/SHIPBroadcaster.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts index 0cbb99b83..839dc658d 100644 --- a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts +++ b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts @@ -93,9 +93,10 @@ export class HTTPSOverlayBroadcastFacilitator implements OverlayBroadcastFacilit const defaultClient = typeof globalThis !== 'undefined' && typeof globalThis.fetch === 'function' ? globalThis.fetch.bind(globalThis) - : undefined - if (!httpClient && !defaultClient) throw new Error('Fetch API is unavailable') - this.httpClient = httpClient ?? defaultClient! + : (async () => { + throw new Error('Fetch API is unavailable') + }) as typeof fetch + this.httpClient = httpClient ?? defaultClient this.allowHTTP = allowHTTP } From fc0a3d97f21de9c11c47ac3286464f75edce2254 Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 14:07:09 -0500 Subject: [PATCH 120/168] fix: defer unavailable fetch failure until send --- packages/sdk/src/overlay-tools/SHIPBroadcaster.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts index 0cbb99b83..839dc658d 100644 --- a/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts +++ b/packages/sdk/src/overlay-tools/SHIPBroadcaster.ts @@ -93,9 +93,10 @@ export class HTTPSOverlayBroadcastFacilitator implements OverlayBroadcastFacilit const defaultClient = typeof globalThis !== 'undefined' && typeof globalThis.fetch === 'function' ? globalThis.fetch.bind(globalThis) - : undefined - if (!httpClient && !defaultClient) throw new Error('Fetch API is unavailable') - this.httpClient = httpClient ?? defaultClient! + : (async () => { + throw new Error('Fetch API is unavailable') + }) as typeof fetch + this.httpClient = httpClient ?? defaultClient this.allowHTTP = allowHTTP } From 9b7c99d41d150fc152f146c98eb5bcef10964fcd Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 14:08:05 -0500 Subject: [PATCH 121/168] test: provide fetch spy target in node --- .../overlay/src/__tests/BASMReconciliation.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts index c984af8ae..b113888a0 100644 --- a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts +++ b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts @@ -78,6 +78,15 @@ function fixture() { engine.logger = { ...console, error: jest.fn() } const submit = jest.spyOn(engine, 'submit').mockResolvedValue({}) const requests: Array<{ path: string; body: Record }> = [] + if (typeof globalThis.fetch !== 'function') { + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + writable: true, + value: async () => { + throw new Error('unexpected network request') + } + }) + } const fetchMock = jest.spyOn(globalThis, 'fetch').mockImplementation(async (url, init) => { const endpoint = new URL(String(url)).pathname requests.push({ From 49e1063feb170e01e91a71f88248276e5cf33acb Mon Sep 17 00:00:00 2001 From: Deggen Date: Mon, 14 Sep 2026 14:08:25 -0500 Subject: [PATCH 122/168] test: match complete admission acknowledgment --- .../overlays/overlay/src/__tests/BASMReconciliation.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts index b113888a0..55d581791 100644 --- a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts +++ b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts @@ -452,7 +452,7 @@ describe('BASM reconciliation evidence binding', () => { f.submit.mockRestore() await expect(f.engine.submit({ beef, topics: [topic] }, undefined, 'historical-tx')).resolves.toEqual({ - [topic]: { outputsToAdmit: [0], coinsToRetain: [] } + [topic]: { outputsToAdmit: [0], coinsToRetain: [], coinsRemoved: [] } }) expect(manager.identifyAdmissibleOutputs).toHaveBeenCalledWith(beef, [], undefined, 'historical-tx') expect(f.storage.insertAppliedTransaction).toHaveBeenCalledWith( From de94acf167bbd9b6134d3211218362957eebb9d7 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 14:08:15 -0500 Subject: [PATCH 123/168] chore(overlays): re-derive release metadata after the ordered merges Refreshes the overlay package documentation dates for the four packages this change bumps and regenerates the derived reference documents so the stack facts and package API migration tables match the merged versions. Co-Authored-By: Claude Fable 5.1 --- .../overlays/overlay-discovery-services.md | 4 ++-- docs/packages/overlays/overlay-express.md | 4 ++-- docs/packages/overlays/overlay.md | 4 ++-- docs/reference/package-api-migrations.md | 24 +++++++++---------- docs/reference/stack-facts.md | 8 +++---- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/packages/overlays/overlay-discovery-services.md b/docs/packages/overlays/overlay-discovery-services.md index dbea79539..1fb0ff1b4 100644 --- a/docs/packages/overlays/overlay-discovery-services.md +++ b/docs/packages/overlays/overlay-discovery-services.md @@ -5,8 +5,8 @@ kind: package domain: overlays npm: '@bsv/overlay-discovery-services' version: '2.2.2' -last_updated: '2026-09-08' -last_verified: '2026-09-08' +last_updated: '2026-09-18' +last_verified: '2026-09-18' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services' status: stable diff --git a/docs/packages/overlays/overlay-express.md b/docs/packages/overlays/overlay-express.md index a2286439f..3af920759 100644 --- a/docs/packages/overlays/overlay-express.md +++ b/docs/packages/overlays/overlay-express.md @@ -5,8 +5,8 @@ kind: package domain: overlays npm: '@bsv/overlay-express' version: '2.6.2' -last_updated: '2026-09-08' -last_verified: '2026-09-08' +last_updated: '2026-09-18' +last_verified: '2026-09-18' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express' status: stable diff --git a/docs/packages/overlays/overlay.md b/docs/packages/overlays/overlay.md index e6195bc2d..1fc0675a6 100644 --- a/docs/packages/overlays/overlay.md +++ b/docs/packages/overlays/overlay.md @@ -5,8 +5,8 @@ kind: package domain: overlays npm: '@bsv/overlay' version: '2.4.0' -last_updated: '2026-09-09' -last_verified: '2026-09-09' +last_updated: '2026-09-18' +last_verified: '2026-09-18' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay' status: stable diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index c712f8aa0..40ec9a8b5 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -42,10 +42,10 @@ and clean-consumer tests remain the executable type authority. | `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | | `@bsv/message-box-client` | `2.4.0` | `2.5.1` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. No migration is required for the bundled payment optimization. Module consumers can use SDK 2.5.0 or later to enable the same optional extension; existing compatible SDK peer versions remain supported. Recipients must advertise already-validated transaction IDs through x-bsv-payment-known-txids, an optional SDK extension rather than a standardized BRC-105 header; services that omit it retain existing payment behavior. Existing return shapes and payment envelopes remain unchanged. Failed or incomplete notification payments stay queued. Resolve uncertain refund-send outcomes before retrying; this patch adds ordering checks, not an exactly-once refund journal. listMessages/listMessagesLite envelope behavior and basket-insertion semantics remain tracked in issue #503. | -| `@bsv/overlay` | `2.2.1` | `2.3.1` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.1` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-express` | `2.5.0` | `2.6.1` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/overlay-topics` | `1.6.10` | `1.8.0` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. | +| `@bsv/overlay` | `2.2.1` | `2.4.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. | +| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.2` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | +| `@bsv/overlay-express` | `2.5.0` | `2.6.2` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | +| `@bsv/overlay-topics` | `1.6.10` | `1.8.1` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | | `@bsv/paymail` | `2.4.2` | `2.4.8` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. Consumers of the former bundled Money Button or Tokenized specification documents must follow the authoritative links in docs/specs/README.md. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/payment-express-middleware` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/sdk` | `2.7.0` | `2.7.1` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No API or wire migration is required. Upgrade to @bsv/sdk 2.7.1 or later. Valid BRC-103 v0.1 peers keep the same message fields, payload bytes, nonces, and signature derivation; authenticated messages whose transport identity differs from the established session now fail closed. | @@ -277,8 +277,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay.md](../packages/overlays/overlay.md) - Source: [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) -- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. -- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. +- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. +- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | @@ -292,8 +292,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-discovery-services.md](../packages/overlays/overlay-discovery-services.md) - Source: [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) -- Release note: Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. -- Migration: Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. +- Release note: Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. +- Migration: Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------------- | ---------------------------------------------------- | @@ -304,8 +304,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-express.md](../packages/overlays/overlay-express.md) - Source: [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) -- Release note: Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. -- Migration: Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. +- Release note: Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. +- Migration: Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------------- | ---------------------------------------------------- | @@ -316,8 +316,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-topics.md](../packages/overlays/overlay-topics.md) - Source: [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) -- Release note: Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. Aligns the UORA v3 reader with the versioned format, including compressed locking keys, exact drop tails, and printable UTF-8 fields. Version 1.8.0 requires verified per-asset admin history and matching stored token ownership for Mandala admission, assigns registrations to their own genesis outpoint, rejects ambiguous linkage indices, and normalizes in-memory sanctions key casing. Retains the complete token-output linkage verification from 1.7.3. -- Migration: Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. +- Release note: Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. Aligns the UORA v3 reader with the versioned format, including compressed locking keys, exact drop tails, and printable UTF-8 fields. Version 1.8.0 requires verified per-asset admin history and matching stored token ownership for Mandala admission, assigns registrations to their own genesis outpoint, rejects ambiguous linkage indices, and normalizes in-memory sanctions key casing. Retains the complete token-output linkage verification from 1.7.3. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. +- Migration: Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | -------------------------------------- | --------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index fae17041d..1e0dbf86d 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -58,10 +58,10 @@ authorized release action. | network | `@bsv/chirp` | `0.1.1` | browser-library | browser-bundler, browser-esm, cli, node-esm | browser, node | `>=22` | [packages/network/chirp](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/chirp) | | network | `@bsv/teranode-listener` | `1.1.5` | node-library | node-esm | node | `>=22` | [packages/network/ts-p2p](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/ts-p2p) | | overlays | `@bsv/gasp` | `1.3.6` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/overlays/gasp-core](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/gasp-core) | -| overlays | `@bsv/overlay` | `2.3.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | -| overlays | `@bsv/overlay-discovery-services` | `2.2.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | -| overlays | `@bsv/overlay-express` | `2.6.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | -| overlays | `@bsv/overlay-topics` | `1.8.0` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | +| overlays | `@bsv/overlay` | `2.4.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | +| overlays | `@bsv/overlay-discovery-services` | `2.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | +| overlays | `@bsv/overlay-express` | `2.6.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | +| overlays | `@bsv/overlay-topics` | `1.8.1` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | | sdk | `@bsv/sdk` | `2.7.1` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | | sdk | `@bsv/verifast` | `0.3.5` | wasm-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global, wasm-worker | browser, node, umd, wasm, worker | `>=22` | [packages/verifast](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/verifast) | | wallet | `@bsv/btms` | `1.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | From 3bd536d8ce76529a5cd53b181b17e5b017086dd3 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 14:24:49 -0500 Subject: [PATCH 124/168] chore(overlays): re-derive release metadata after the ordered merges Apply this PR's own SemVer impact on top of what the predecessor branch leaves, and restore the cumulative release prose the branch had regressed. - Versions: @bsv/overlay 2.5.0, @bsv/overlay-express 2.7.0, @bsv/overlay-discovery-services 2.2.3, @bsv/overlay-topics 1.8.2, with matching repository-health baselines and docs frontmatter. - governance/package-release-notes.json: keep main's and the predecessor's published versions and cumulative text, and append only this PR's BASM sentences for the four overlay packages. - docs/packages/sdk/bsv-sdk.md: make the Merkle-path candidate note version-neutral so it no longer names a superseded release. - Regenerate docs/reference/package-api-migrations.md, stack-facts.md and service-operations.md. Co-Authored-By: Claude Fable 5.1 --- .../overlays/overlay-discovery-services.md | 2 +- docs/packages/overlays/overlay-express.md | 2 +- docs/packages/overlays/overlay-topics.md | 2 +- docs/packages/overlays/overlay.md | 2 +- docs/packages/sdk/bsv-sdk.md | 2 +- docs/reference/package-api-migrations.md | 24 +++++++++---------- docs/reference/stack-facts.md | 8 +++---- governance/package-release-notes.json | 16 ++++++------- governance/repository-health/baselines.json | 8 +++---- .../overlay-discovery-services/package.json | 2 +- packages/overlays/overlay/package.json | 2 +- packages/overlays/topics/package.json | 2 +- 12 files changed, 36 insertions(+), 36 deletions(-) diff --git a/docs/packages/overlays/overlay-discovery-services.md b/docs/packages/overlays/overlay-discovery-services.md index 1fb0ff1b4..0aa1bbd52 100644 --- a/docs/packages/overlays/overlay-discovery-services.md +++ b/docs/packages/overlays/overlay-discovery-services.md @@ -4,7 +4,7 @@ title: '@bsv/overlay-discovery-services' kind: package domain: overlays npm: '@bsv/overlay-discovery-services' -version: '2.2.2' +version: '2.2.3' last_updated: '2026-09-18' last_verified: '2026-09-18' review_cadence_days: 30 diff --git a/docs/packages/overlays/overlay-express.md b/docs/packages/overlays/overlay-express.md index 3af920759..24c1ff7ea 100644 --- a/docs/packages/overlays/overlay-express.md +++ b/docs/packages/overlays/overlay-express.md @@ -4,7 +4,7 @@ title: '@bsv/overlay-express' kind: package domain: overlays npm: '@bsv/overlay-express' -version: '2.6.2' +version: '2.7.0' last_updated: '2026-09-18' last_verified: '2026-09-18' review_cadence_days: 30 diff --git a/docs/packages/overlays/overlay-topics.md b/docs/packages/overlays/overlay-topics.md index ac636f130..63205b89f 100644 --- a/docs/packages/overlays/overlay-topics.md +++ b/docs/packages/overlays/overlay-topics.md @@ -4,7 +4,7 @@ title: '@bsv/overlay-topics' kind: package domain: overlays npm: '@bsv/overlay-topics' -version: '1.8.1' +version: '1.8.2' last_updated: '2026-09-18' last_verified: '2026-09-18' review_cadence_days: 30 diff --git a/docs/packages/overlays/overlay.md b/docs/packages/overlays/overlay.md index 1fc0675a6..312d65026 100644 --- a/docs/packages/overlays/overlay.md +++ b/docs/packages/overlays/overlay.md @@ -4,7 +4,7 @@ title: '@bsv/overlay' kind: package domain: overlays npm: '@bsv/overlay' -version: '2.4.0' +version: '2.5.0' last_updated: '2026-09-18' last_verified: '2026-09-18' review_cadence_days: 30 diff --git a/docs/packages/sdk/bsv-sdk.md b/docs/packages/sdk/bsv-sdk.md index b5bd8911c..24a6a630e 100644 --- a/docs/packages/sdk/bsv-sdk.md +++ b/docs/packages/sdk/bsv-sdk.md @@ -15,7 +15,7 @@ repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk' # @bsv/sdk -The unpublished 2.5.0 candidate corrects BUMP offset arithmetic above 32 bits +The current unpublished candidate corrects BUMP offset arithmetic above 32 bits through `Number.MAX_SAFE_INTEGER`, preserving existing wire encodings. Root calculation, extraction, combination and trimming use the same exact numeric domain; malformed non-integer and unsafe offsets fail explicitly. No consumer diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 40ec9a8b5..4bd878df7 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -42,10 +42,10 @@ and clean-consumer tests remain the executable type authority. | `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | | `@bsv/message-box-client` | `2.4.0` | `2.5.1` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. No migration is required for the bundled payment optimization. Module consumers can use SDK 2.5.0 or later to enable the same optional extension; existing compatible SDK peer versions remain supported. Recipients must advertise already-validated transaction IDs through x-bsv-payment-known-txids, an optional SDK extension rather than a standardized BRC-105 header; services that omit it retain existing payment behavior. Existing return shapes and payment envelopes remain unchanged. Failed or incomplete notification payments stay queued. Resolve uncertain refund-send outcomes before retrying; this patch adds ordering checks, not an exactly-once refund journal. listMessages/listMessagesLite envelope behavior and basket-insertion semantics remain tracked in issue #503. | -| `@bsv/overlay` | `2.2.1` | `2.4.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. | -| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.2` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | -| `@bsv/overlay-express` | `2.5.0` | `2.6.2` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | -| `@bsv/overlay-topics` | `1.6.10` | `1.8.1` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. | +| `@bsv/overlay` | `2.2.1` | `2.5.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. | +| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.3` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update. | +| `@bsv/overlay-express` | `2.5.0` | `2.7.0` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. Valid legacy JSON shapes, numeric-string heights, public CORS, topic-free raw transactions, configured limits and synchronization defaults remain supported. No storage migration is required. Malformed/coercive inputs are rejected and missing BASM engine/storage capabilities return HTTP 400 with code BASM_UNSUPPORTED. Existing header providers remain valid without a blockTransactionCount field. | +| `@bsv/overlay-topics` | `1.6.10` | `1.8.2` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update. | | `@bsv/paymail` | `2.4.2` | `2.4.8` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. Consumers of the former bundled Money Button or Tokenized specification documents must follow the authoritative links in docs/specs/README.md. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/payment-express-middleware` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/sdk` | `2.7.0` | `2.7.1` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No API or wire migration is required. Upgrade to @bsv/sdk 2.7.1 or later. Valid BRC-103 v0.1 peers keep the same message fields, payload bytes, nonces, and signature derivation; authenticated messages whose transport identity differs from the established session now fail closed. | @@ -277,8 +277,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay.md](../packages/overlays/overlay.md) - Source: [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) -- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. -- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. +- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Claimed admitted-list indices are bound to the compound path even when every remote txid is already local, and inclusion uses chain-tracker root/height rather than coinbase maturity. +- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | @@ -292,8 +292,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-discovery-services.md](../packages/overlays/overlay-discovery-services.md) - Source: [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) -- Release note: Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. -- Migration: Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. +- Release note: Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. Advances the packed overlay dependency candidate for BASM validation hardening without changing this package runtime behavior. +- Migration: Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------------- | ---------------------------------------------------- | @@ -304,8 +304,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-express.md](../packages/overlays/overlay-express.md) - Source: [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) -- Release note: Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. -- Migration: Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. +- Release note: Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. Validates BASM heights, hashes and txid sets; exposes unsupported capabilities explicitly and accepts an optional trusted full-block count from injected canonical header resolvers. +- Migration: Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. Valid legacy JSON shapes, numeric-string heights, public CORS, topic-free raw transactions, configured limits and synchronization defaults remain supported. No storage migration is required. Malformed/coercive inputs are rejected and missing BASM engine/storage capabilities return HTTP 400 with code BASM_UNSUPPORTED. Existing header providers remain valid without a blockTransactionCount field. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------------- | ---------------------------------------------------- | @@ -316,8 +316,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-topics.md](../packages/overlays/overlay-topics.md) - Source: [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) -- Release note: Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. Aligns the UORA v3 reader with the versioned format, including compressed locking keys, exact drop tails, and printable UTF-8 fields. Version 1.8.0 requires verified per-asset admin history and matching stored token ownership for Mandala admission, assigns registrations to their own genesis outpoint, rejects ambiguous linkage indices, and normalizes in-memory sanctions key casing. Retains the complete token-output linkage verification from 1.7.3. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. -- Migration: Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. +- Release note: Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. Aligns the UORA v3 reader with the versioned format, including compressed locking keys, exact drop tails, and printable UTF-8 fields. Version 1.8.0 requires verified per-asset admin history and matching stored token ownership for Mandala admission, assigns registrations to their own genesis outpoint, rejects ambiguous linkage indices, and normalizes in-memory sanctions key casing. Retains the complete token-output linkage verification from 1.7.3. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. Advances the packed overlay dependency candidate for BASM validation hardening without changing this package runtime behavior. +- Migration: Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | -------------------------------------- | --------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 1e0dbf86d..7f8878d17 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -58,10 +58,10 @@ authorized release action. | network | `@bsv/chirp` | `0.1.1` | browser-library | browser-bundler, browser-esm, cli, node-esm | browser, node | `>=22` | [packages/network/chirp](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/chirp) | | network | `@bsv/teranode-listener` | `1.1.5` | node-library | node-esm | node | `>=22` | [packages/network/ts-p2p](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/ts-p2p) | | overlays | `@bsv/gasp` | `1.3.6` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/overlays/gasp-core](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/gasp-core) | -| overlays | `@bsv/overlay` | `2.4.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | -| overlays | `@bsv/overlay-discovery-services` | `2.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | -| overlays | `@bsv/overlay-express` | `2.6.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | -| overlays | `@bsv/overlay-topics` | `1.8.1` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | +| overlays | `@bsv/overlay` | `2.5.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | +| overlays | `@bsv/overlay-discovery-services` | `2.2.3` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | +| overlays | `@bsv/overlay-express` | `2.7.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | +| overlays | `@bsv/overlay-topics` | `1.8.2` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | | sdk | `@bsv/sdk` | `2.7.1` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | | sdk | `@bsv/verifast` | `0.3.5` | wasm-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global, wasm-worker | browser, node, umd, wasm, worker | `>=22` | [packages/verifast](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/verifast) | | wallet | `@bsv/btms` | `1.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index f61810131..069a72bc1 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -126,29 +126,29 @@ "name": "@bsv/overlay", "publishedVersion": "2.2.1", "releaseType": "minor", - "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection.", - "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included." + "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Claimed admitted-list indices are bound to the compound path even when every remote txid is already local, and inclusion uses chain-tracker root/height rather than coinbase maturity.", + "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included." }, { "name": "@bsv/overlay-discovery-services", "publishedVersion": "2.1.1", "releaseType": "minor", - "summary": "Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior.", - "migration": "Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged." + "summary": "Extends WalletAdvertiser to TerraTestNet with chain-isolated SHIP/SLAP resolution and TTN wallet services. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. Advances the packed overlay dependency candidate for BASM validation hardening without changing this package runtime behavior.", + "migration": "Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update." }, { "name": "@bsv/overlay-express", "publishedVersion": "2.5.0", "releaseType": "minor", - "summary": "Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior.", - "migration": "Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged." + "summary": "Adds a TerraTestNet runtime mode with TTN discovery roots, wallet advertising, and explicit ChainTracks requirements while retaining Arcade-first propagation. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. Validates BASM heights, hashes and txid sets; exposes unsupported capabilities explicitly and accepts an optional trusted full-block count from injected canonical header resolvers.", + "migration": "Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. Valid legacy JSON shapes, numeric-string heights, public CORS, topic-free raw transactions, configured limits and synchronization defaults remain supported. No storage migration is required. Malformed/coercive inputs are rejected and missing BASM engine/storage capabilities return HTTP 400 with code BASM_UNSUPPORTED. Existing header providers remain valid without a blockTransactionCount field." }, { "name": "@bsv/overlay-topics", "publishedVersion": "1.6.10", "releaseType": "minor", - "summary": "Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. Aligns the UORA v3 reader with the versioned format, including compressed locking keys, exact drop tails, and printable UTF-8 fields. Version 1.8.0 requires verified per-asset admin history and matching stored token ownership for Mandala admission, assigns registrations to their own genesis outpoint, rejects ambiguous linkage indices, and normalizes in-memory sanctions key casing. Retains the complete token-output linkage verification from 1.7.3. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior.", - "migration": "Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged." + "summary": "Adds persistent first-writer reservations for UMP presentation and recovery hashes, aborts provisional claims after strict broadcast failure, keeps confirmed owners protected until successor indexing, retries transient initialization, marks one-time legacy bootstrap, and returns the newest bounded legacy candidates for verified lineage or an operator pin. It also retains the collection-index resilience and opt-in repair controls from the prior candidate. Standardizes first-party author metadata on the current BSV Association name. Aligns the UORA v3 reader with the versioned format, including compressed locking keys, exact drop tails, and printable UTF-8 fields. Version 1.8.0 requires verified per-asset admin history and matching stored token ownership for Mandala admission, assigns registrations to their own genesis outpoint, rejects ambiguous linkage indices, and normalizes in-memory sanctions key casing. Retains the complete token-output linkage verification from 1.7.3. Updates the packed workspace dependency candidate for the additive overlay persistence contracts without changing this package runtime behavior. Advances the packed overlay dependency candidate for BASM validation hardening without changing this package runtime behavior.", + "migration": "Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update." }, { "name": "@bsv/paymail", diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index d4aa601dc..b439a1bed 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -318,10 +318,10 @@ "@bsv/payment-express-middleware": "2.1.6", "@bsv/teranode-listener": "1.1.5", "@bsv/gasp": "1.3.6", - "@bsv/overlay": "2.4.0", - "@bsv/overlay-discovery-services": "2.2.2", - "@bsv/overlay-express": "2.6.2", - "@bsv/overlay-topics": "1.8.1", + "@bsv/overlay": "2.5.0", + "@bsv/overlay-discovery-services": "2.2.3", + "@bsv/overlay-express": "2.7.0", + "@bsv/overlay-topics": "1.8.2", "@bsv/sdk": "2.7.1", "@bsv/verifast": "0.3.5", "@bsv/btms": "1.2.2", diff --git a/packages/overlays/overlay-discovery-services/package.json b/packages/overlays/overlay-discovery-services/package.json index 610bb5080..fa870977a 100644 --- a/packages/overlays/overlay-discovery-services/package.json +++ b/packages/overlays/overlay-discovery-services/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-discovery-services", - "version": "2.2.2", + "version": "2.2.3", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/overlay/package.json b/packages/overlays/overlay/package.json index 406b7e56e..de23ed672 100644 --- a/packages/overlays/overlay/package.json +++ b/packages/overlays/overlay/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay", - "version": "2.4.0", + "version": "2.5.0", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/topics/package.json b/packages/overlays/topics/package.json index 4500673e1..bf49900ff 100644 --- a/packages/overlays/topics/package.json +++ b/packages/overlays/topics/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-topics", - "version": "1.8.1", + "version": "1.8.2", "sideEffects": false, "engines": { "node": ">=22" From 7c5ca8f5175c20b28fc4f2651899f7b90ec972de Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 14:59:21 -0500 Subject: [PATCH 125/168] chore(overlays): re-derive release metadata after the ordered merges Bumps the overlay family's own version fields on top of what #517-#520 already carry: @bsv/overlay 2.6.0, @bsv/overlay-express 2.7.1, @bsv/overlay-discovery-services 2.2.4, @bsv/overlay-topics 1.8.3. No @bsv/sdk or @bsv/wallet-toolbox version bump is included; that release decision stays with the maintainer. Updates governance/repository-health/baselines.json's publicPackageVersions for the four bumped packages, and regenerates docs/reference/package-api-migrations.md and docs/reference/stack-facts.md (pnpm docs:facts) plus each package's docs page frontmatter so they match. governance/package-release-notes.json's cumulative summary and migration prose for @bsv/overlay already carries this PR's own opt-in Mongo AdmissionStorage adapter / Engine.submit() admission-commit sentence, layered onto #519/#520's BASM prose during the predecessor merge; no further edit is needed here. Co-Authored-By: Claude Fable 5.1 --- docs/packages/overlays/overlay-discovery-services.md | 2 +- docs/packages/overlays/overlay-express.md | 2 +- docs/packages/overlays/overlay-topics.md | 2 +- docs/packages/overlays/overlay.md | 2 +- docs/reference/package-api-migrations.md | 12 ++++++------ docs/reference/stack-facts.md | 8 ++++---- governance/repository-health/baselines.json | 8 ++++---- .../overlays/overlay-discovery-services/package.json | 2 +- packages/overlays/overlay-express/package.json | 2 +- packages/overlays/overlay/package.json | 2 +- packages/overlays/topics/package.json | 2 +- 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/packages/overlays/overlay-discovery-services.md b/docs/packages/overlays/overlay-discovery-services.md index 0aa1bbd52..20c3fda4d 100644 --- a/docs/packages/overlays/overlay-discovery-services.md +++ b/docs/packages/overlays/overlay-discovery-services.md @@ -4,7 +4,7 @@ title: '@bsv/overlay-discovery-services' kind: package domain: overlays npm: '@bsv/overlay-discovery-services' -version: '2.2.3' +version: '2.2.4' last_updated: '2026-09-18' last_verified: '2026-09-18' review_cadence_days: 30 diff --git a/docs/packages/overlays/overlay-express.md b/docs/packages/overlays/overlay-express.md index 24c1ff7ea..aa7388b91 100644 --- a/docs/packages/overlays/overlay-express.md +++ b/docs/packages/overlays/overlay-express.md @@ -4,7 +4,7 @@ title: '@bsv/overlay-express' kind: package domain: overlays npm: '@bsv/overlay-express' -version: '2.7.0' +version: '2.7.1' last_updated: '2026-09-18' last_verified: '2026-09-18' review_cadence_days: 30 diff --git a/docs/packages/overlays/overlay-topics.md b/docs/packages/overlays/overlay-topics.md index 63205b89f..f6a066e02 100644 --- a/docs/packages/overlays/overlay-topics.md +++ b/docs/packages/overlays/overlay-topics.md @@ -4,7 +4,7 @@ title: '@bsv/overlay-topics' kind: package domain: overlays npm: '@bsv/overlay-topics' -version: '1.8.2' +version: '1.8.3' last_updated: '2026-09-18' last_verified: '2026-09-18' review_cadence_days: 30 diff --git a/docs/packages/overlays/overlay.md b/docs/packages/overlays/overlay.md index 312d65026..942d47265 100644 --- a/docs/packages/overlays/overlay.md +++ b/docs/packages/overlays/overlay.md @@ -4,7 +4,7 @@ title: '@bsv/overlay' kind: package domain: overlays npm: '@bsv/overlay' -version: '2.5.0' +version: '2.6.0' last_updated: '2026-09-18' last_verified: '2026-09-18' review_cadence_days: 30 diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 4bd878df7..fffd1ed0e 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -42,10 +42,10 @@ and clean-consumer tests remain the executable type authority. | `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | | `@bsv/message-box-client` | `2.4.0` | `2.5.1` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. No migration is required for the bundled payment optimization. Module consumers can use SDK 2.5.0 or later to enable the same optional extension; existing compatible SDK peer versions remain supported. Recipients must advertise already-validated transaction IDs through x-bsv-payment-known-txids, an optional SDK extension rather than a standardized BRC-105 header; services that omit it retain existing payment behavior. Existing return shapes and payment envelopes remain unchanged. Failed or incomplete notification payments stay queued. Resolve uncertain refund-send outcomes before retrying; this patch adds ordering checks, not an exactly-once refund journal. listMessages/listMessagesLite envelope behavior and basket-insertion semantics remain tracked in issue #503. | -| `@bsv/overlay` | `2.2.1` | `2.5.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. | -| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.3` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update. | -| `@bsv/overlay-express` | `2.5.0` | `2.7.0` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. Valid legacy JSON shapes, numeric-string heights, public CORS, topic-free raw transactions, configured limits and synchronization defaults remain supported. No storage migration is required. Malformed/coercive inputs are rejected and missing BASM engine/storage capabilities return HTTP 400 with code BASM_UNSUPPORTED. Existing header providers remain valid without a blockTransactionCount field. | -| `@bsv/overlay-topics` | `1.6.10` | `1.8.2` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update. | +| `@bsv/overlay` | `2.2.1` | `2.6.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Knex/SQL submit remains the default path; Engine uses commitAdmission only when storage advertises a complete overlay-admission-v1 provider. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. | +| `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.4` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update. | +| `@bsv/overlay-express` | `2.5.0` | `2.7.1` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. Valid legacy JSON shapes, numeric-string heights, public CORS, topic-free raw transactions, configured limits and synchronization defaults remain supported. No storage migration is required. Malformed/coercive inputs are rejected and missing BASM engine/storage capabilities return HTTP 400 with code BASM_UNSUPPORTED. Existing header providers remain valid without a blockTransactionCount field. | +| `@bsv/overlay-topics` | `1.6.10` | `1.8.3` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update. | | `@bsv/paymail` | `2.4.2` | `2.4.8` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. Consumers of the former bundled Money Button or Tokenized specification documents must follow the authoritative links in docs/specs/README.md. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/payment-express-middleware` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/sdk` | `2.7.0` | `2.7.1` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No API or wire migration is required. Upgrade to @bsv/sdk 2.7.1 or later. Valid BRC-103 v0.1 peers keep the same message fields, payload bytes, nonces, and signature derivation; authenticated messages whose transport identity differs from the established session now fail closed. | @@ -277,8 +277,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay.md](../packages/overlays/overlay.md) - Source: [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) -- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Claimed admitted-list indices are bound to the compound path even when every remote txid is already local, and inclusion uses chain-tracker root/height rather than coinbase maturity. -- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. +- Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Claimed admitted-list indices are bound to the compound path even when every remote txid is already local, and inclusion uses chain-tracker root/height rather than coinbase maturity. Adds an opt-in Mongo AdmissionStorage adapter and Engine submit path that uses commitAdmission when overlay-admission-v1 is advertised, with enlisted indexes or a durable projection outbox and majority ACK. +- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Knex/SQL submit remains the default path; Engine uses commitAdmission only when storage advertises a complete overlay-admission-v1 provider. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 7f8878d17..970147e1a 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -58,10 +58,10 @@ authorized release action. | network | `@bsv/chirp` | `0.1.1` | browser-library | browser-bundler, browser-esm, cli, node-esm | browser, node | `>=22` | [packages/network/chirp](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/chirp) | | network | `@bsv/teranode-listener` | `1.1.5` | node-library | node-esm | node | `>=22` | [packages/network/ts-p2p](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/ts-p2p) | | overlays | `@bsv/gasp` | `1.3.6` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/overlays/gasp-core](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/gasp-core) | -| overlays | `@bsv/overlay` | `2.5.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | -| overlays | `@bsv/overlay-discovery-services` | `2.2.3` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | -| overlays | `@bsv/overlay-express` | `2.7.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | -| overlays | `@bsv/overlay-topics` | `1.8.2` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | +| overlays | `@bsv/overlay` | `2.6.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) | +| overlays | `@bsv/overlay-discovery-services` | `2.2.4` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | +| overlays | `@bsv/overlay-express` | `2.7.1` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | +| overlays | `@bsv/overlay-topics` | `1.8.3` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | | sdk | `@bsv/sdk` | `2.7.1` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | | sdk | `@bsv/verifast` | `0.3.5` | wasm-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global, wasm-worker | browser, node, umd, wasm, worker | `>=22` | [packages/verifast](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/verifast) | | wallet | `@bsv/btms` | `1.2.2` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index b439a1bed..56191b7df 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -318,10 +318,10 @@ "@bsv/payment-express-middleware": "2.1.6", "@bsv/teranode-listener": "1.1.5", "@bsv/gasp": "1.3.6", - "@bsv/overlay": "2.5.0", - "@bsv/overlay-discovery-services": "2.2.3", - "@bsv/overlay-express": "2.7.0", - "@bsv/overlay-topics": "1.8.2", + "@bsv/overlay": "2.6.0", + "@bsv/overlay-discovery-services": "2.2.4", + "@bsv/overlay-express": "2.7.1", + "@bsv/overlay-topics": "1.8.3", "@bsv/sdk": "2.7.1", "@bsv/verifast": "0.3.5", "@bsv/btms": "1.2.2", diff --git a/packages/overlays/overlay-discovery-services/package.json b/packages/overlays/overlay-discovery-services/package.json index fa870977a..9276562c6 100644 --- a/packages/overlays/overlay-discovery-services/package.json +++ b/packages/overlays/overlay-discovery-services/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-discovery-services", - "version": "2.2.3", + "version": "2.2.4", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/overlay-express/package.json b/packages/overlays/overlay-express/package.json index 27c767b98..63394cc6d 100644 --- a/packages/overlays/overlay-express/package.json +++ b/packages/overlays/overlay-express/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-express", - "version": "2.7.0", + "version": "2.7.1", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/overlay/package.json b/packages/overlays/overlay/package.json index de23ed672..c352f34e7 100644 --- a/packages/overlays/overlay/package.json +++ b/packages/overlays/overlay/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay", - "version": "2.5.0", + "version": "2.6.0", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/overlays/topics/package.json b/packages/overlays/topics/package.json index bf49900ff..c4d620f1f 100644 --- a/packages/overlays/topics/package.json +++ b/packages/overlays/topics/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/overlay-topics", - "version": "1.8.2", + "version": "1.8.3", "sideEffects": false, "engines": { "node": ">=22" From 07ea459955e9f1f9bfe55400bb185b90476df70e Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 15:13:28 -0500 Subject: [PATCH 126/168] fix(sdk): keep the evidence branch esbuild budget at its measured ceiling The main integration merge raised the SDK esbuild raw ceiling to 600000, a value that belongs to the lookup discovery change (#518). This branch measured 590000; restore it so the diff carries only its own budget. Co-Authored-By: Claude Fable 5.1 --- packages/sdk/browser-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index f72f855e8..6705fcce5 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -21,7 +21,7 @@ "brotli": 165000 }, "esbuild": { - "raw": 600000, + "raw": 590000, "gzip": 180000, "brotli": 150000 } From 6e748c653189d6f7b7a7667de8988f947911e65c Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 15:13:50 -0500 Subject: [PATCH 127/168] fix(sdk): scope lookup discovery docs and budget to this change Restore the eleven ts2md reference pages for modules this change does not touch to the incoming copies; the generator run had imported unrelated drift that already exists on main. Keep the regenerated overlay-tools page. Reinstate this change's own 600000 esbuild raw ceiling after restacking on the corrected evidence branch. Co-Authored-By: Claude Fable 5.1 --- packages/sdk/browser-budget.json | 2 +- packages/sdk/docs/reference/auth.md | 28 +- packages/sdk/docs/reference/compat.md | 43 +- packages/sdk/docs/reference/identity.md | 112 +--- packages/sdk/docs/reference/kvstore.md | 56 +- packages/sdk/docs/reference/messages.md | 2 +- packages/sdk/docs/reference/primitives.md | 666 +++++++++------------- packages/sdk/docs/reference/registry.md | 3 +- packages/sdk/docs/reference/remittance.md | 10 + packages/sdk/docs/reference/script.md | 252 +++----- packages/sdk/docs/reference/storage.md | 4 +- packages/sdk/docs/reference/wallet.md | 457 +++++---------- 12 files changed, 560 insertions(+), 1075 deletions(-) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index 6705fcce5..f72f855e8 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -21,7 +21,7 @@ "brotli": 165000 }, "esbuild": { - "raw": 590000, + "raw": 600000, "gzip": 180000, "brotli": 150000 } diff --git a/packages/sdk/docs/reference/auth.md b/packages/sdk/docs/reference/auth.md index ad1db90be..9e351ee89 100644 --- a/packages/sdk/docs/reference/auth.md +++ b/packages/sdk/docs/reference/auth.md @@ -67,31 +67,9 @@ export interface PeerSession { lastUpdate: number; certificatesRequired?: boolean; certificatesValidated?: boolean; - certificatePolicy?: RequestedCertificateSet; - pendingCertificateRequests?: Record; } ``` -See also: [RequestedCertificateSet](./auth.md#interface-requestedcertificateset) - -#### Property certificatePolicy - -Local handshake policy snapshot. Session stores must retain this field; never sent on the wire. - -```ts -certificatePolicy?: RequestedCertificateSet -``` -See also: [RequestedCertificateSet](./auth.md#interface-requestedcertificateset) - -#### Property pendingCertificateRequests - -Locally issued standalone requests, keyed by their nonce. Not a wire correlation field. - -```ts -pendingCertificateRequests?: Record -``` -See also: [RequestedCertificateSet](./auth.md#interface-requestedcertificateset) - Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- @@ -833,11 +811,7 @@ Argument Details #### Method listenForCertificatesReceived -Registers an observer for certificates received from peers, not an acceptance hook. -Local certificate validation is committed and its waiters are released before observers -run. Throwing rejects message handling and stops subsequent observers; it does not -roll back validation or revoke the session. Apply acceptance policy through the locally -requested certificate set and explicit application authorization before protected work. +Registers a callback to listen for certificates received from peers. ```ts listenForCertificatesReceived(callback: (senderPublicKey: string, certs: VerifiableCertificate[]) => void | Promise): number diff --git a/packages/sdk/docs/reference/compat.md b/packages/sdk/docs/reference/compat.md index 0cb8231b1..da7a70b2c 100644 --- a/packages/sdk/docs/reference/compat.md +++ b/packages/sdk/docs/reference/compat.md @@ -470,7 +470,6 @@ export default class Mnemonic { public toString(): string public toSeed(passphrase?: string): number[] public entropy2Mnemonic(buf: number[]): this - public toEntropy(): number[] public check(): boolean public mnemonic2Seed(passphrase = ""): this public isValid(passphrase = ""): boolean @@ -652,11 +651,6 @@ Argument Details + **mnemonic** + The mnemonic phrase as a string. -Throws - -If the mnemonic does not pass BIP-39 validation -(unknown words, invalid length, or bad checksum). - #### Method fromString Static method to create a Mnemonic instance from a mnemonic string. @@ -748,22 +742,6 @@ Returns The binary representation of the mnemonic and seed. -#### Method toEntropy - -Recovers the original entropy bytes from the instance's mnemonic phrase. - -```ts -public toEntropy(): number[] -``` - -Returns - -The entropy buffer that was originally used to generate the mnemonic. - -Throws - -If the mnemonic is invalid or contains unknown words. - #### Method toSeed Converts the mnemonic to a seed. @@ -859,9 +837,20 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Variable: magicHash ```ts -magicHash = (messageBuf: number[]): number[] => computeMagicHash(messageBuf) +magicHash = (messageBuf: number[]): number[] => { + const bw = new Writer(); + bw.writeVarIntNum(prefix.length); + bw.write(toArray(prefix, "utf8")); + bw.writeVarIntNum(messageBuf.length); + bw.write(messageBuf); + const buf = bw.toArray(); + const hashBuf = Hash.hash256(buf); + return hashBuf; +} ``` +See also: [Writer](./primitives.md#class-writer), [hash256](./primitives.md#variable-hash256), [toArray](./primitives.md#variable-toarray) + Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- @@ -869,7 +858,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ```ts sign = (message: number[], privateKey: PrivateKey, mode: "raw" | "base64" = "base64"): Signature | string => { - const hashBuf = computeMagicHash(message); + const hashBuf = magicHash(message); const sig = ECDSA.sign(new BigNumber(hashBuf), privateKey, true); if (mode === "raw") { return sig; @@ -880,7 +869,7 @@ sign = (message: number[], privateKey: PrivateKey, mode: "raw" | "base64" = "bas } ``` -See also: [BigNumber](./primitives.md#class-bignumber), [PrivateKey](./primitives.md#class-privatekey), [Signature](./primitives.md#class-signature) +See also: [BigNumber](./primitives.md#class-bignumber), [PrivateKey](./primitives.md#class-privatekey), [Signature](./primitives.md#class-signature), [magicHash](./compat.md#variable-magichash) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -889,12 +878,12 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ```ts verify = (message: number[], sig: Signature, pubKey: PublicKey): boolean => { - const hashBuf = computeMagicHash(message); + const hashBuf = magicHash(message); return ECDSA.verify(new BigNumber(hashBuf), sig, pubKey); } ``` -See also: [BigNumber](./primitives.md#class-bignumber), [PublicKey](./primitives.md#class-publickey), [Signature](./primitives.md#class-signature) +See also: [BigNumber](./primitives.md#class-bignumber), [PublicKey](./primitives.md#class-publickey), [Signature](./primitives.md#class-signature), [magicHash](./compat.md#variable-magichash) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) diff --git a/packages/sdk/docs/reference/identity.md b/packages/sdk/docs/reference/identity.md index 3550c83ad..0f1a4fffa 100644 --- a/packages/sdk/docs/reference/identity.md +++ b/packages/sdk/docs/reference/identity.md @@ -8,8 +8,6 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | --- | | [DisplayableIdentity](#interface-displayableidentity) | | [IdentityClientOptions](#interface-identityclientoptions) | -| [ResolveByAttributesOptions](#interface-resolvebyattributesoptions) | -| [ResolveByIdentityKeyOptions](#interface-resolvebyidentitykeyoptions) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -40,59 +38,11 @@ export interface IdentityClientOptions { keyID: string; tokenAmount: number; outputIndex: number; - networkPreset?: LookupNetworkPreset; + networkPreset?: "mainnet" | "testnet" | "teratestnet" | "local"; } ``` -See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset), [WalletProtocol](./wallet.md#type-walletprotocol) - -#### Property networkPreset - -Override wallet-reported testnet routing for overlays such as TerraTestNet. - -```ts -networkPreset?: LookupNetworkPreset -``` -See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset) - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- -### Interface: ResolveByAttributesOptions - -```ts -export interface ResolveByAttributesOptions { - useContacts?: boolean; - overrideWithContacts?: boolean; - parallel?: boolean; -} -``` - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- -### Interface: ResolveByIdentityKeyOptions - -```ts -export interface ResolveByIdentityKeyOptions { - useContacts?: boolean; - overrideWithContacts?: boolean; - parallel?: boolean; -} -``` - -#### Property useContacts - -Opt-in to consulting personal contacts before/alongside the overlay. Default `false`. - -Most callers (including any client without a populated contacts basket) pay no benefit -from the contacts path and incur its setup cost. Set `true` only in UI contexts where -the user has likely saved contacts and a local cache hit is preferable to a fresh overlay -answer. - -```ts -useContacts?: boolean -``` +See also: [WalletProtocol](./wallet.md#type-walletprotocol) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -123,18 +73,17 @@ See also: [Contact](./identity.md#type-contact), [DisplayableIdentity](./identit #### Method getContacts -Load all records from the contacts basket. - -Concurrent calls share a single in-flight load (no thundering herd). After -the basket has been observed empty once, subsequent calls return `[]` -synchronously without hitting the wallet — until `forceRefresh` is passed -or a contact is saved/removed. +Load all records from the contacts basket ```ts async getContacts(identityKey?: PubKeyHex, forceRefresh = false, limit = 1000): Promise ``` See also: [Contact](./identity.md#type-contact), [PubKeyHex](./wallet.md#type-pubkeyhex) +Returns + +A promise that resolves with an array of contacts + Argument Details + **identityKey** @@ -182,21 +131,19 @@ IdentityClient lets you discover who others are, and let the world know who you ```ts export class IdentityClient { - constructor(wallet?: WalletInterface, options: Partial = {}, private readonly originator?: OriginatorDomainNameStringUnder250Bytes) + constructor(wallet?: WalletInterface, options?: Partial, private readonly originator?: OriginatorDomainNameStringUnder250Bytes) async publiclyRevealAttributes(certificate: WalletCertificate, fieldsToReveal: CertificateFieldNameUnder50Bytes[]): Promise - async resolveByIdentityKey(args: DiscoverByIdentityKeyArgs, opts: boolean | ResolveByIdentityKeyOptions = false): Promise - async resolveByAttributes(args: DiscoverByAttributesArgs, opts: boolean | ResolveByAttributesOptions = false): Promise + async resolveByIdentityKey(args: DiscoverByIdentityKeyArgs, overrideWithContacts = true): Promise + async resolveByAttributes(args: DiscoverByAttributesArgs, overrideWithContacts = true): Promise async revokeCertificateRevelation(serialNumber: Base64String): Promise public async getContacts(identityKey?: PubKeyHex, forceRefresh = false, limit = 1000): Promise public async saveContact(contact: DisplayableIdentity, metadata?: Record): Promise public async removeContact(identityKey: PubKeyHex): Promise - static async parseIdentities(certs: IdentityCertificate[]): Promise - static async parseIdentitiesWithOverrides(certs: IdentityCertificate[], contactByKey: Map): Promise static parseIdentity(identityToParse: IdentityCertificate): DisplayableIdentity } ``` -See also: [Base64String](./wallet.md#type-base64string), [BroadcastFailure](./transaction.md#interface-broadcastfailure), [BroadcastResponse](./transaction.md#interface-broadcastresponse), [CertificateFieldNameUnder50Bytes](./wallet.md#type-certificatefieldnameunder50bytes), [Contact](./identity.md#type-contact), [DiscoverByAttributesArgs](./wallet.md#interface-discoverbyattributesargs), [DiscoverByIdentityKeyArgs](./wallet.md#interface-discoverbyidentitykeyargs), [DisplayableIdentity](./identity.md#interface-displayableidentity), [IdentityCertificate](./wallet.md#interface-identitycertificate), [IdentityClientOptions](./identity.md#interface-identityclientoptions), [OriginatorDomainNameStringUnder250Bytes](./wallet.md#type-originatordomainnamestringunder250bytes), [PubKeyHex](./wallet.md#type-pubkeyhex), [ResolveByAttributesOptions](./identity.md#interface-resolvebyattributesoptions), [ResolveByIdentityKeyOptions](./identity.md#interface-resolvebyidentitykeyoptions), [WalletCertificate](./wallet.md#interface-walletcertificate), [WalletInterface](./wallet.md#interface-walletinterface) +See also: [Base64String](./wallet.md#type-base64string), [BroadcastFailure](./transaction.md#interface-broadcastfailure), [BroadcastResponse](./transaction.md#interface-broadcastresponse), [CertificateFieldNameUnder50Bytes](./wallet.md#type-certificatefieldnameunder50bytes), [Contact](./identity.md#type-contact), [DEFAULT_IDENTITY_CLIENT_OPTIONS](./identity.md#variable-default_identity_client_options), [DiscoverByAttributesArgs](./wallet.md#interface-discoverbyattributesargs), [DiscoverByIdentityKeyArgs](./wallet.md#interface-discoverbyidentitykeyargs), [DisplayableIdentity](./identity.md#interface-displayableidentity), [IdentityCertificate](./wallet.md#interface-identitycertificate), [OriginatorDomainNameStringUnder250Bytes](./wallet.md#type-originatordomainnamestringunder250bytes), [PubKeyHex](./wallet.md#type-pubkeyhex), [WalletCertificate](./wallet.md#interface-walletcertificate), [WalletInterface](./wallet.md#interface-walletinterface) #### Method getContacts @@ -279,42 +226,43 @@ Argument Details #### Method resolveByAttributes +Resolves displayable identity certificates by specific identity attributes, issued by a trusted entity. + ```ts -async resolveByAttributes(args: DiscoverByAttributesArgs, opts: boolean | ResolveByAttributesOptions = false): Promise +async resolveByAttributes(args: DiscoverByAttributesArgs, overrideWithContacts = true): Promise ``` -See also: [DiscoverByAttributesArgs](./wallet.md#interface-discoverbyattributesargs), [DisplayableIdentity](./identity.md#interface-displayableidentity), [ResolveByAttributesOptions](./identity.md#interface-resolvebyattributesoptions) +See also: [DiscoverByAttributesArgs](./wallet.md#interface-discoverbyattributesargs), [DisplayableIdentity](./identity.md#interface-displayableidentity) + +Returns + +The promise resolves to displayable identities. Argument Details + **args** + Attributes and optional parameters used to discover certificates. -+ **opts** - + Boolean (legacy) or options object. Boolean `true` ≡ `{ useContacts: true }`. ++ **overrideWithContacts** + + Whether to override the results with personal contacts if available. #### Method resolveByIdentityKey -Resolves displayable identity certificates issued to a given identity key. - -**Default behavior (changed): contacts are NOT consulted.** Most clients have no -contacts saved locally, so the previous "contacts-first" default paid setup cost for no -gain. Pass `{ useContacts: true }` to opt in — appropriate when you know the user has -saved contacts and prefers a local hit over a fresh overlay answer. - -When `useContacts: true`: - - Default short-circuits: if a contact matches, the overlay is skipped entirely. - - `{ parallel: true }` fires contacts and overlay in parallel; contact wins on hit. +Resolves displayable identity certificates, issued to a given identity key by a trusted certifier. ```ts -async resolveByIdentityKey(args: DiscoverByIdentityKeyArgs, opts: boolean | ResolveByIdentityKeyOptions = false): Promise +async resolveByIdentityKey(args: DiscoverByIdentityKeyArgs, overrideWithContacts = true): Promise ``` -See also: [DiscoverByIdentityKeyArgs](./wallet.md#interface-discoverbyidentitykeyargs), [DisplayableIdentity](./identity.md#interface-displayableidentity), [ResolveByIdentityKeyOptions](./identity.md#interface-resolvebyidentitykeyoptions) +See also: [DiscoverByIdentityKeyArgs](./wallet.md#interface-discoverbyidentitykeyargs), [DisplayableIdentity](./identity.md#interface-displayableidentity) + +Returns + +The promise resolves to displayable identities. Argument Details + **args** + Arguments for requesting the discovery based on the identity key. -+ **opts** - + Boolean (legacy) or options object. Boolean `true` ≡ `{ useContacts: true }`. ++ **overrideWithContacts** + + Whether to override the results with personal contacts if available. #### Method revokeCertificateRevelation diff --git a/packages/sdk/docs/reference/kvstore.md b/packages/sdk/docs/reference/kvstore.md index 5f3beb69c..6fb6c3cd5 100644 --- a/packages/sdk/docs/reference/kvstore.md +++ b/packages/sdk/docs/reference/kvstore.md @@ -48,10 +48,7 @@ export interface KVStoreConfig { topics?: string[]; originator?: string; wallet?: WalletInterface; - networkPreset?: LookupNetworkPreset; - lookupResolver?: LookupResolver; - hostOverrides?: Record; - slapTrackers?: string[]; + networkPreset?: "mainnet" | "testnet" | "teratestnet" | "local"; acceptDelayedBroadcast?: boolean; overlayBroadcast?: boolean; tokenSetDescription?: string; @@ -60,7 +57,7 @@ export interface KVStoreConfig { } ``` -See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset), [LookupResolver](./overlay-tools.md#class-lookupresolver), [WalletInterface](./wallet.md#interface-walletinterface), [WalletProtocol](./wallet.md#type-walletprotocol) +See also: [WalletInterface](./wallet.md#interface-walletinterface), [WalletProtocol](./wallet.md#type-walletprotocol) #### Property acceptDelayedBroadcast @@ -70,44 +67,13 @@ Whether to accept delayed broadcast acceptDelayedBroadcast?: boolean ``` -#### Property hostOverrides - -Per-service overlay host overrides (`serviceName -> hosts`), applied when -the store builds its default lookup resolver. This pins which hosts answer -*lookup* queries for a given service (e.g. read lookups via `ls_kvstore`), -instead of discovering them via SLAP. - -Note this does not by itself pin the *broadcast* target: writes are -submitted to the hosts that the `ls_ship` SHIP lookup returns, so an -`ls_ship` override only changes which tracker answers — the broadcast host -is whatever advertisements that lookup names. To force writes to a specific -backend, use a resolver / SHIP setup whose `ls_ship` results return the -desired host. Ignored when `lookupResolver` is supplied. - -```ts -hostOverrides?: Record -``` - -#### Property lookupResolver - -A pre-built lookup resolver to use for all overlay queries — both reads and -write-host (SHIP) discovery. When provided, it takes precedence and -`hostOverrides` / `slapTrackers` are ignored for resolver construction. -Use this to fully control overlay host resolution. - -```ts -lookupResolver?: LookupResolver -``` -See also: [LookupResolver](./overlay-tools.md#class-lookupresolver) - #### Property networkPreset Network preset for overlay services ```ts -networkPreset?: LookupNetworkPreset +networkPreset?: "mainnet" | "testnet" | "teratestnet" | "local" ``` -See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset) #### Property originator @@ -150,15 +116,6 @@ Service name for overlay submission serviceName?: string ``` -#### Property slapTrackers - -Override the SLAP trackers used by the default lookup resolver. Ignored when -`lookupResolver` is supplied. - -```ts -slapTrackers?: string[] -``` - #### Property tokenAmount Amount of satoshis for each token @@ -292,8 +249,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Interface: KVStoreQuery Query parameters for KVStore lookups from overlay services. -Must include at least one selector: key, controller, protocolID, or non-empty tags. -Pagination and ordering fields only refine selector-based lookups. +Used when searching for existing key-value pairs in the network. ```ts export interface KVStoreQuery { @@ -658,9 +614,9 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( kvStoreInterpreter: InterpreterFunction = async (transaction: Transaction, outputIndex: number, ctx?: KVContext): Promise => { try { const output = transaction.outputs[outputIndex]; - if (output?.lockingScript == null) + if (output == null || output.lockingScript == null) return undefined; - if (ctx?.key == null) + if (ctx == null || ctx.key == null) return undefined; const decoded = PushDrop.decode(output.lockingScript); const expectedFieldCount = Object.keys(kvProtocol).length; diff --git a/packages/sdk/docs/reference/messages.md b/packages/sdk/docs/reference/messages.md index 8db915d13..35eea667b 100644 --- a/packages/sdk/docs/reference/messages.md +++ b/packages/sdk/docs/reference/messages.md @@ -134,7 +134,7 @@ verify = (message: number[], sig: number[], recipient?: PrivateKey): boolean => const verifierRest = reader.read(32); const verifierDER = toHex([verifierFirst, ...verifierRest]); if (typeof recipient !== "object") { - throw new TypeError(`This signature can only be verified with knowledge of a specific private key. The associated public key is: ${verifierDER}`); + throw new Error(`This signature can only be verified with knowledge of a specific private key. The associated public key is: ${verifierDER}`); } const recipientDER = recipient.toPublicKey().encode(true, "hex") as string; if (verifierDER !== recipientDER) { diff --git a/packages/sdk/docs/reference/primitives.md b/packages/sdk/docs/reference/primitives.md index f851db4a0..02566f574 100644 --- a/packages/sdk/docs/reference/primitives.md +++ b/packages/sdk/docs/reference/primitives.md @@ -6,8 +6,6 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | | | --- | -| [AsyncCryptoBackend](#interface-asynccryptobackend) | -| [DigestVerification](#interface-digestverification) | | [JacobianPointBI](#interface-jacobianpointbi) | | [SignatureHashCache](#interface-signaturehashcache) | @@ -15,47 +13,6 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( --- -### Interface: AsyncCryptoBackend - -Optional high-performance implementation of generic secp256k1 primitives. - -Implementations must treat a returned result as authoritative. The SDK only -falls back before selecting a backend: when it is absent, cold, or does not -advertise the requested operation. - -```ts -export interface AsyncCryptoBackend { - preload: () => Promise; - isReady: () => boolean; - supportsCrypto: (operation: AsyncCryptoOperation) => boolean; - signDigest: (privateKey: Uint8Array, digest: Uint8Array) => Promise; - verifyDigest: (publicKey: Uint8Array, digest: Uint8Array, signature: Uint8Array) => Promise; - verifyDigestBatch: (items: readonly DigestVerification[]) => Promise; - publicKeyFromPrivate: (privateKey: Uint8Array) => Promise; - multiplyPublicKey: (publicKey: Uint8Array, scalar: Uint8Array) => Promise; - tweakPublicKeyAdd: (publicKey: Uint8Array, tweak: Uint8Array) => Promise; - tweakPrivateKeyAdd: (privateKey: Uint8Array, tweak: Uint8Array) => Promise; -} -``` - -See also: [AsyncCryptoOperation](./primitives.md#type-asynccryptooperation), [DigestVerification](./primitives.md#interface-digestverification) - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- -### Interface: DigestVerification - -```ts -export interface DigestVerification { - publicKey: Uint8Array; - digest: Uint8Array; - signature: Uint8Array; -} -``` - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- ### Interface: JacobianPointBI ```ts @@ -71,12 +28,6 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( --- ### Interface: SignatureHashCache -Reusable BIP143 hash components for one immutable transaction context. - -Callers sharing a cache across inputs must not mutate transaction prevouts, -sequences, outputs, or other signed fields until that signing or verification -pass is complete. Create a fresh cache for a changed transaction context. - ```ts export interface SignatureHashCache { hashPrevouts?: number[]; @@ -150,7 +101,7 @@ export default class BigNumber { static readonly groupSizes: number[] static readonly groupBases: number[] static readonly wordSize: number = 26; - public red: ReductionContext | null = null; + public red: ReductionContext | null; public get negative(): number public set negative(val: number) public get words(): number[] @@ -165,23 +116,27 @@ export default class BigNumber { clone(): BigNumber expand(size: number): this strip(): this - normSign(): this + normSign(): this { if (this._magnitude === 0n) + this._sign = 0; return this; } inspect(): string toString(base: number | "hex" = 10, padding: number = 1): string toNumber(): number toBigInt(): bigint toJSON(): string toArray(endian: "le" | "be" = "be", length?: number): number[] - bitLength(): number + bitLength(): number { if (this._magnitude === 0n) + return 0; return this._magnitude.toString(2).length; } static toBitArray(num: BigNumber): Array<0 | 1> toBitArray(): Array<0 | 1> zeroBits(): number - byteLength(): number + byteLength(): number { if (this._magnitude === 0n) + return 0; return Math.ceil(this.bitLength() / 8); } toTwos(width: number): BigNumber fromTwos(width: number): BigNumber isNeg(): boolean neg(): BigNumber - ineg(): this + ineg(): this { if (this._magnitude !== 0n) + this._sign = this._sign === 1 ? 0 : 1; return this; } iuor(num: BigNumber): this iuand(num: BigNumber): this iuxor(num: BigNumber): this @@ -196,7 +151,10 @@ export default class BigNumber { uxor(num: BigNumber): BigNumber inotn(width: number): this notn(width: number): BigNumber - setn(bit: number, val: any): this + setn(bit: number, val: any): this { this.assert(typeof bit === "number" && bit >= 0); const Bb = BigInt(bit); if (val === 1 || val === true) + this._magnitude |= (1n << Bb); + else + this._magnitude &= ~(1n << Bb); const wnb = Math.floor(bit / BigNumber.wordSize) + 1; this._nominalWordLength = Math.max(this._nominalWordLength, wnb); this._finishInitialization(); return this.strip(); } iadd(num: BigNumber): this add(num: BigNumber): BigNumber isub(num: BigNumber): this @@ -246,9 +204,15 @@ export default class BigNumber { andln(num: number): number bincn(bit: number): this isZero(): boolean - cmpn(num: number): CompareResult - cmp(num: BigNumber): CompareResult - ucmp(num: BigNumber): CompareResult + cmpn(num: number): 1 | 0 | -1 { this.assert(Math.abs(num) <= BigNumber.MAX_IMULN_ARG, "Number is too big"); const tV = this._getSignedValue(); const nV = BigInt(num); if (tV < nV) + return -1; if (tV > nV) + return 1; return 0; } + cmp(num: BigNumber): 1 | 0 | -1 { const tV = this._getSignedValue(); const nV = num._getSignedValue(); if (tV < nV) + return -1; if (tV > nV) + return 1; return 0; } + ucmp(num: BigNumber): 1 | 0 | -1 { if (this._magnitude < num._magnitude) + return -1; if (this._magnitude > num._magnitude) + return 1; return 0; } gtn(num: number): boolean gt(num: BigNumber): boolean gten(num: number): boolean @@ -313,7 +277,7 @@ Argument Details Reduction context of the big number. ```ts -public red: ReductionContext | null = null +public red: ReductionContext | null ``` See also: [ReductionContext](./primitives.md#class-reductioncontext) @@ -362,7 +326,8 @@ Argument Details Calculates the number of bits required to represent the BigNumber. ```ts -bitLength(): number +bitLength(): number { if (this._magnitude === 0n) + return 0; return this._magnitude.toString(2).length; } ``` Returns @@ -374,7 +339,8 @@ The bit length of the BigNumber. Calculates the number of bytes required to represent the BigNumber. ```ts -byteLength(): number +byteLength(): number { if (this._magnitude === 0n) + return 0; return Math.ceil(this.bitLength() / 8); } ``` Returns @@ -799,19 +765,19 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ```ts export default class Curve { - p!: BigNumber; - red!: ReductionContext; - redN!: BigNumber | null; - zero!: BigNumber; - one!: BigNumber; - two!: BigNumber; - g!: Point; - n!: BigNumber; - a!: BigNumber; - b!: BigNumber; - tinv!: BigNumber; - zeroA!: boolean; - threeA!: boolean; + p: BigNumber; + red: ReductionContext; + redN: BigNumber | null; + zero: BigNumber; + one: BigNumber; + two: BigNumber; + g: Point; + n: BigNumber; + a: BigNumber; + b: BigNumber; + tinv: BigNumber; + zeroA: boolean; + threeA: boolean; endo: { beta: BigNumber; lambda: BigNumber; @@ -820,21 +786,21 @@ export default class Curve { b: BigNumber; }>; } | undefined; - _endoWnafT1!: BigNumber[]; - _endoWnafT2!: BigNumber[]; - _wnafT1!: BigNumber[]; - _wnafT2!: BigNumber[]; - _wnafT3!: BigNumber[]; - _wnafT4!: BigNumber[]; - _bitLength!: number; + _endoWnafT1: BigNumber[]; + _endoWnafT2: BigNumber[]; + _wnafT1: BigNumber[]; + _wnafT2: BigNumber[]; + _wnafT3: BigNumber[]; + _wnafT4: BigNumber[]; + _bitLength: number; static assert(expression: unknown, message: string = "Elliptic curve assertion failed"): void getNAF(num: BigNumber, w: number, bits: number): number[] getJSF(k1: BigNumber, k2: BigNumber): number[][] - static cachedProperty(obj: any, name: string, computer: (this: any) => unknown): void + static cachedProperty(obj, name: string, computer): void static parseBytes(bytes: string | number[]): number[] static intFromLE(bytes: number[]): BigNumber constructor() - _getEndomorphism(conf: EndomorphismConfig): { + _getEndomorphism(conf): { beta: BigNumber; lambda: BigNumber; basis: Array<{ @@ -991,7 +957,7 @@ export default class JacobianPoint extends BasePoint { y: BigNumber; z: BigNumber; zOne: boolean; - constructor(x: JacobianCoord, y: JacobianCoord, z: JacobianCoord) + constructor(x: string | BigNumber | null, y: string | BigNumber | null, z: string | BigNumber | null) toP(): Point neg(): JacobianPoint add(p: JacobianPoint): JacobianPoint @@ -1012,8 +978,9 @@ See also: [BasePoint](./primitives.md#class-basepoint), [BigNumber](./primitives Constructs a new `JacobianPoint` instance. ```ts -constructor(x: JacobianCoord, y: JacobianCoord, z: JacobianCoord) +constructor(x: string | BigNumber | null, y: string | BigNumber | null, z: string | BigNumber | null) ``` +See also: [BigNumber](./primitives.md#class-bignumber) Argument Details @@ -1321,8 +1288,8 @@ const k256 = new K256(); ```ts export default class K256 extends Mersenne { constructor() - override split(input: BigNumber, output: BigNumber): void - override imulK(num: BigNumber): BigNumber + split(input: BigNumber, output: BigNumber): void + imulK(num: BigNumber): BigNumber } ``` @@ -1349,7 +1316,7 @@ Multiplies a BigNumber ('num') with the constant 'K' in-place and returns the re 'K' is equal to 0x1000003d1 or in decimal representation: [ 64, 977 ]. ```ts -override imulK(num: BigNumber): BigNumber +imulK(num: BigNumber): BigNumber ``` See also: [BigNumber](./primitives.md#class-bignumber) @@ -1375,7 +1342,7 @@ Splits a BigNumber into a new BigNumber based on specific computation rules. This method modifies the input and output big numbers. ```ts -override split(input: BigNumber, output: BigNumber): void +split(input: BigNumber, output: BigNumber): void ``` See also: [BigNumber](./primitives.md#class-bignumber) @@ -1588,11 +1555,11 @@ export default class MontgomoryMethod extends ReductionContext { rinv: BigNumber; minv: BigNumber; constructor(m: BigNumber | "k256") - override convertTo(num: BigNumber): BigNumber - override convertFrom(num: BigNumber): BigNumber - override imul(a: BigNumber, b: BigNumber): BigNumber - override mul(a: BigNumber, b: BigNumber): BigNumber - override invm(a: BigNumber): BigNumber + convertTo(num: BigNumber): BigNumber + convertFrom(num: BigNumber): BigNumber + imul(a: BigNumber, b: BigNumber): BigNumber + mul(a: BigNumber, b: BigNumber): BigNumber + invm(a: BigNumber): BigNumber } ``` @@ -1659,7 +1626,7 @@ shift: number Converts a number from the Montgomery domain back to the original domain. ```ts -override convertFrom(num: BigNumber): BigNumber +convertFrom(num: BigNumber): BigNumber ``` See also: [BigNumber](./primitives.md#class-bignumber) @@ -1684,7 +1651,7 @@ const convertedNum = montMethod.convertFrom(num); Converts a number into the Montgomery domain. ```ts -override convertTo(num: BigNumber): BigNumber +convertTo(num: BigNumber): BigNumber ``` See also: [BigNumber](./primitives.md#class-bignumber) @@ -1709,7 +1676,7 @@ const convertedNum = montMethod.convertTo(num); Performs an in-place multiplication of two numbers in the Montgomery domain. ```ts -override imul(a: BigNumber, b: BigNumber): BigNumber +imul(a: BigNumber, b: BigNumber): BigNumber ``` See also: [BigNumber](./primitives.md#class-bignumber) @@ -1736,7 +1703,7 @@ const product = montMethod.imul(a, b); Calculates the modular multiplicative inverse of a number in the Montgomery domain. ```ts -override invm(a: BigNumber): BigNumber +invm(a: BigNumber): BigNumber ``` See also: [BigNumber](./primitives.md#class-bignumber) @@ -1761,7 +1728,7 @@ const inverse = montMethod.invm(a); Performs the multiplication of two numbers in the Montgomery domain. ```ts -override mul(a: BigNumber, b: BigNumber): BigNumber +mul(a: BigNumber, b: BigNumber): BigNumber ``` See also: [BigNumber](./primitives.md#class-bignumber) @@ -1800,12 +1767,12 @@ export default class Point extends BasePoint { static _assertOnCurve(p: Point): Point static fromDER(bytes: number[]): Point static fromString(str: string): Point - static fromX(x: PointInput, odd: boolean): Point + static fromX(x: BigNumber | number | number[] | string, odd: boolean): Point static fromJSON(obj: string | any[], isRed: boolean): Point - constructor(x: PointInput | null, y: PointInput | null, isRed: boolean = true) + constructor(x: BigNumber | number | number[] | string | null, y: BigNumber | number | number[] | string | null, isRed: boolean = true) validate(): boolean encode(compact: boolean = true, enc?: "hex"): number[] | string - override toString(): string + toString(): string toJSON(): [ BigNumber | null, BigNumber | null, @@ -1826,8 +1793,8 @@ export default class Point extends BasePoint { dbl(): Point getX(): BigNumber getY(): BigNumber - mul(k: PointInput): Point - mulCT(k: PointInput): Point + mul(k: BigNumber | number | number[] | string): Point + mulCT(k: BigNumber | number | number[] | string): Point mulAdd(k1: BigNumber, p2: Point, k2: BigNumber): Point jmulAdd(k1: BigNumber, p2: Point, k2: BigNumber): JPoint eq(p: Point): boolean @@ -1842,8 +1809,9 @@ See also: [BasePoint](./primitives.md#class-basepoint), [BigNumber](./primitives #### Constructor ```ts -constructor(x: PointInput | null, y: PointInput | null, isRed: boolean = true) +constructor(x: BigNumber | number | number[] | string | null, y: BigNumber | number | number[] | string | null, isRed: boolean = true) ``` +See also: [BigNumber](./primitives.md#class-bignumber) Argument Details @@ -2116,9 +2084,9 @@ Generates a point from an x coordinate and a boolean indicating whether the corr y coordinate is odd. ```ts -static fromX(x: PointInput, odd: boolean): Point +static fromX(x: BigNumber | number | number[] | string, odd: boolean): Point ``` -See also: [Point](./primitives.md#class-point) +See also: [BigNumber](./primitives.md#class-bignumber), [Point](./primitives.md#class-point) Returns @@ -2248,9 +2216,9 @@ const result = p1.jmulAdd(2, p2, 3); Multiplies this Point by a scalar value, returning a new Point. ```ts -mul(k: PointInput): Point +mul(k: BigNumber | number | number[] | string): Point ``` -See also: [Point](./primitives.md#class-point) +See also: [BigNumber](./primitives.md#class-bignumber), [Point](./primitives.md#class-point) Returns @@ -2376,7 +2344,7 @@ Converts the point coordinates to a hexadecimal string. A wrapper method for encode. Byte 0x02 or 0x03 is used as prefix based on the 'y' coordinate being even or odd respectively. ```ts -override toString(): string +toString(): string ``` Returns @@ -2478,8 +2446,8 @@ create a corresponding public key and derive a shared secret from a public key. ```ts export default class PrivateKey extends BigNumber { static fromRandom(): PrivateKey - static override fromString(str: string, base: number | "hex" = "hex"): PrivateKey - static override fromHex(str: string): PrivateKey + static fromString(str: string, base: number | "hex" = "hex"): PrivateKey + static fromHex(str: string): PrivateKey static fromWif(wif: string, prefixLength: number = 1): PrivateKey constructor(number: BigNumber | number | string | number[] = 0, base: number | "be" | "le" | "hex" = 10, endian: "be" | "le" = "be", modN: "apply" | "nocheck" | "error" = "apply") checkInField(): { @@ -2492,10 +2460,10 @@ export default class PrivateKey extends BigNumber { toPublicKey(): PublicKey toWif(prefix: number[] = [128]): string toAddress(prefix: number[] | string = [0]): string - override toHex(): string - override toString(base: number | "hex" = "hex", padding: number = 64): string + toHex(): string + toString(base: number | "hex" = "hex", padding: number = 64): string deriveSharedSecret(key: PublicKey): Point - deriveChild(publicKey: PublicKey, invoiceNumber: string, cacheSharedSecret?: (priv: PrivateKey, pub: Point, point: Point) => void, retrieveCachedSharedSecret?: (priv: PrivateKey, pub: Point) => Point | undefined): PrivateKey + deriveChild(publicKey: PublicKey, invoiceNumber: string, cacheSharedSecret?: ((priv: PrivateKey, pub: Point, point: Point) => void), retrieveCachedSharedSecret?: ((priv: PrivateKey, pub: Point) => (Point | undefined))): PrivateKey toKeyShares(threshold: number, totalShares: number): KeyShares toBackupShares(threshold: number, totalShares: number): string[] static fromBackupShares(shares: string[]): PrivateKey @@ -2578,7 +2546,7 @@ peer authentication will require a versioned, breaking change. Derives a child key with BRC-42. ```ts -deriveChild(publicKey: PublicKey, invoiceNumber: string, cacheSharedSecret?: (priv: PrivateKey, pub: Point, point: Point) => void, retrieveCachedSharedSecret?: (priv: PrivateKey, pub: Point) => Point | undefined): PrivateKey +deriveChild(publicKey: PublicKey, invoiceNumber: string, cacheSharedSecret?: ((priv: PrivateKey, pub: Point, point: Point) => void), retrieveCachedSharedSecret?: ((priv: PrivateKey, pub: Point) => (Point | undefined))): PrivateKey ``` See also: [Point](./primitives.md#class-point), [PrivateKey](./primitives.md#class-privatekey), [PublicKey](./primitives.md#class-publickey) @@ -2652,7 +2620,7 @@ const recoveredKey = PrivateKey.fromBackupShares([share1, share2]) Generates a private key from a hexadecimal string. ```ts -static override fromHex(str: string): PrivateKey +static fromHex(str: string): PrivateKey ``` See also: [PrivateKey](./primitives.md#class-privatekey) @@ -2713,7 +2681,7 @@ const privateKey = PrivateKey.fromRandom(); Generates a private key from a string. ```ts -static override fromString(str: string, base: number | "hex" = "hex"): PrivateKey +static fromString(str: string, base: number | "hex" = "hex"): PrivateKey ``` See also: [PrivateKey](./primitives.md#class-privatekey) @@ -2842,7 +2810,7 @@ Argument Details Converts this PrivateKey to a hexadecimal string. ```ts -override toHex(): string +toHex(): string ``` Returns @@ -2919,7 +2887,7 @@ function toString() { [native code] } Converts this PrivateKey to a string representation. ```ts -override toString(base: number | "hex" = "hex", padding: number = 64): string +toString(base: number | "hex" = "hex", padding: number = 64): string ``` Returns @@ -3006,15 +2974,15 @@ The class comes with static methods to generate PublicKey instances from private ```ts export default class PublicKey extends Point { static fromPrivateKey(key: PrivateKey): PublicKey - static override fromString(str: string): PublicKey - static override fromDER(bytes: number[]): PublicKey + static fromString(str: string): PublicKey + static fromDER(bytes: number[]): PublicKey constructor(x: Point | BigNumber | number | number[] | string | null, y: BigNumber | number | number[] | string | null = null, isRed: boolean = true) deriveSharedSecret(priv: PrivateKey): Point verify(msg: number[] | string, sig: Signature, enc?: "hex" | "utf8"): boolean toDER(enc?: "hex" | undefined): number[] | string toHash(enc?: "hex"): number[] | string toAddress(prefix: number[] | string = [0]): string - deriveChild(privateKey: PrivateKey, invoiceNumber: string, cacheSharedSecret?: (priv: PrivateKey, pub: Point, point: Point) => void, retrieveCachedSharedSecret?: (priv: PrivateKey, pub: Point) => Point | undefined): PublicKey + deriveChild(privateKey: PrivateKey, invoiceNumber: string, cacheSharedSecret?: ((priv: PrivateKey, pub: Point, point: Point) => void), retrieveCachedSharedSecret?: ((priv: PrivateKey, pub: Point) => (Point | undefined))): PublicKey static fromMsgHashAndCompactSignature(msgHash: BigNumber, signature: number[] | string, enc?: "hex" | "base64"): PublicKey } ``` @@ -3049,7 +3017,7 @@ new PublicKey('abc123', 'def456'); Derives a child key with BRC-42. ```ts -deriveChild(privateKey: PrivateKey, invoiceNumber: string, cacheSharedSecret?: (priv: PrivateKey, pub: Point, point: Point) => void, retrieveCachedSharedSecret?: (priv: PrivateKey, pub: Point) => Point | undefined): PublicKey +deriveChild(privateKey: PrivateKey, invoiceNumber: string, cacheSharedSecret?: ((priv: PrivateKey, pub: Point, point: Point) => void), retrieveCachedSharedSecret?: ((priv: PrivateKey, pub: Point) => (Point | undefined))): PublicKey ``` See also: [Point](./primitives.md#class-point), [PrivateKey](./primitives.md#class-privatekey), [PublicKey](./primitives.md#class-publickey) @@ -3103,7 +3071,7 @@ const sharedSecret = myPubKey.deriveSharedSecret(myPrivKey) Static factory method to create a PublicKey instance from a number array. ```ts -static override fromDER(bytes: number[]): PublicKey +static fromDER(bytes: number[]): PublicKey ``` See also: [PublicKey](./primitives.md#class-publickey) @@ -3186,7 +3154,7 @@ const myPubKey = PublicKey.fromPrivateKey(myPrivKey) Static factory method to create a PublicKey instance from a string. ```ts -static override fromString(str: string): PublicKey +static fromString(str: string): PublicKey ``` See also: [PublicKey](./primitives.md#class-publickey) @@ -3386,9 +3354,6 @@ export class ReaderUint8Array { constructor(bin: Uint8Array | number[] = new Uint8Array(0), pos: number = 0) public eof(): boolean public read(len = this.length): Uint8Array - public readView(len = this.length - this.pos): Uint8Array - public skip(len: number): void - public remaining(): number public readReverse(len = this.length): Uint8Array public readUInt8(): number public readInt8(): number @@ -3411,14 +3376,6 @@ export class ReaderUint8Array { See also: [BigNumber](./primitives.md#class-bignumber), [Reader](./primitives.md#class-reader) -#### Method skip - -Advances without allocating. - -```ts -public skip(len: number): void -``` - Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- @@ -4040,7 +3997,7 @@ export class SHA1HMAC { outer: SHA1; blockSize = 64; constructor(key: number[] | string) - update(msg: number[] | string, enc?: "hex"): this + update(msg: number[] | string, enc?: "hex"): SHA1HMAC digest(): number[] digestHex(): string } @@ -4067,7 +4024,7 @@ const sha256 = new SHA256(); ```ts export class SHA256 { constructor() - update(msg: HashInput, enc?: "hex" | "utf8"): this + update(msg: Uint8Array | number[] | string, enc?: "hex" | "utf8"): this digest(): number[] digestHex(): string } @@ -4088,8 +4045,8 @@ This class also uses the SHA-256 cryptographic hash algorithm that produces a 25 export class SHA256HMAC { blockSize = 64; outSize = 32; - constructor(key: HashInput) - update(msg: HashInput, enc?: "hex"): this + constructor(key: Uint8Array | number[] | string) + update(msg: Uint8Array | number[] | string, enc?: "hex"): SHA256HMAC digest(): number[] digestHex(): string } @@ -4104,7 +4061,7 @@ If the key size is larger than the blockSize, it is digested using SHA-256. If the key size is less than the blockSize, it is padded with zeroes. ```ts -constructor(key: HashInput) +constructor(key: Uint8Array | number[] | string) ``` Argument Details @@ -4175,8 +4132,9 @@ let hashedMessage = myHMAC.digestHex(); Updates the `SHA256HMAC` object with part of the message to be hashed. ```ts -update(msg: HashInput, enc?: "hex"): this +update(msg: Uint8Array | number[] | string, enc?: "hex"): SHA256HMAC ``` +See also: [SHA256HMAC](./primitives.md#class-sha256hmac) Returns @@ -4214,7 +4172,7 @@ const sha512 = new SHA512(); ```ts export class SHA512 { constructor() - update(msg: HashInput, enc?: "hex" | "utf8"): this + update(msg: number[] | string, enc?: "hex" | "utf8"): this digest(): number[] digestHex(): string } @@ -4235,8 +4193,8 @@ This class also uses the SHA-512 cryptographic hash algorithm that produces a 51 export class SHA512HMAC { blockSize = 128; outSize = 32; - constructor(key: HashInput) - update(msg: HashInput, enc?: "hex" | "utf8"): this + constructor(key: Uint8Array | number[] | string) + update(msg: Uint8Array | number[] | string, enc?: "hex" | "utf8"): SHA512HMAC digest(): number[] digestHex(): string } @@ -4251,7 +4209,7 @@ If the key size is larger than the blockSize, it is digested using SHA-512. If the key size is less than the blockSize, it is padded with zeroes. ```ts -constructor(key: HashInput) +constructor(key: Uint8Array | number[] | string) ``` Argument Details @@ -4322,8 +4280,9 @@ let hashedMessage = myHMAC.digestHex(); Updates the `SHA512HMAC` object with part of the message to be hashed. ```ts -update(msg: HashInput, enc?: "hex" | "utf8"): this +update(msg: Uint8Array | number[] | string, enc?: "hex" | "utf8"): SHA512HMAC ``` +See also: [SHA512HMAC](./primitives.md#class-sha512hmac) Returns @@ -4989,7 +4948,6 @@ export default class TransactionSignature extends Signature { static formatBip143(params: TransactionSignatureFormatParams): Uint8Array static format(params: TransactionSignatureFormatParams): number[] static formatBytes(params: TransactionSignatureFormatParams): Uint8Array - static usesOtdaSingleBug(params: TransactionSignatureFormatParams): boolean static fromChecksigFormat(buf: number[]): TransactionSignature constructor(r: BigNumber, s: BigNumber, scope: number) public hasLowS(): boolean @@ -5109,7 +5067,6 @@ export class WriterUint8Array { toUint8Array(): Uint8Array toArray(): number[] toUint8ArrayZeroCopy(): Uint8Array - reserve(additionalBytes: number): void write(bytes: WriterChunk): this writeReverse(buf: WriterChunk): this writeUInt8(value: number): this @@ -5141,14 +5098,6 @@ Returns the current length of written data getLength(): number ``` -#### Method reserve - -Ensures room for `additionalBytes` without changing the written length. - -```ts -reserve(additionalBytes: number): void -``` - #### Method reset Resets the writer to empty state (reuses the buffer) @@ -5192,17 +5141,14 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | | | | --- | --- | -| [AES](#function-aes) | [readyAsyncCryptoBackend](#function-readyasynccryptobackend) | -| [AESGCM](#function-aesgcm) | [realHtonl](#function-realhtonl) | -| [AESGCMDecrypt](#function-aesgcmdecrypt) | [red](#function-red) | -| [assertValidHex](#function-assertvalidhex) | [registerAsyncCryptoBackend](#function-registerasynccryptobackend) | +| [AES](#function-aes) | [normalizeHex](#function-normalizehex) | +| [AESGCM](#function-aesgcm) | [pbkdf2](#function-pbkdf2) | +| [AESGCMDecrypt](#function-aesgcmdecrypt) | [realHtonl](#function-realhtonl) | +| [assertValidHex](#function-assertvalidhex) | [red](#function-red) | | [base64ToArray](#function-base64toarray) | [swapBytes32](#function-swapbytes32) | | [constantTimeEquals](#function-constanttimeequals) | [toArray](#function-toarray) | | [ghash](#function-ghash) | [toBase64](#function-tobase64) | -| [htonl](#function-htonl) | [unregisterAsyncCryptoBackend](#function-unregisterasynccryptobackend) | -| [isAsyncCryptoDigest](#function-isasynccryptodigest) | [validateAsyncCryptoBytes](#function-validateasynccryptobytes) | -| [normalizeHex](#function-normalizehex) | [verifyNotNull](#function-verifynotnull) | -| [pbkdf2](#function-pbkdf2) | | +| [htonl](#function-htonl) | [verifyNotNull](#function-verifynotnull) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -5328,17 +5274,6 @@ export function htonl(w: number): number Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) ---- -### Function: isAsyncCryptoDigest - -True when a caller supplied the canonical 32-byte digest representation. - -```ts -export function isAsyncCryptoDigest(digest: readonly number[]): boolean -``` - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - --- ### Function: normalizeHex @@ -5376,20 +5311,6 @@ Argument Details Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) ---- -### Function: readyAsyncCryptoBackend - -Returns a warm backend supporting `operation`. A cold backend is prepared in -the background while the current call retains the existing JavaScript path. - -```ts -export function readyAsyncCryptoBackend(operation: AsyncCryptoOperation): AsyncCryptoBackend | undefined -``` - -See also: [AsyncCryptoBackend](./primitives.md#interface-asynccryptobackend), [AsyncCryptoOperation](./primitives.md#type-asynccryptooperation) - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - --- ### Function: realHtonl @@ -5435,19 +5356,6 @@ export function red(x: bigint): bigint Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) ---- -### Function: registerAsyncCryptoBackend - -Installs an optional process/page-wide backend for opportunistic SDK use. - -```ts -export function registerAsyncCryptoBackend(backend: AsyncCryptoBackend): void -``` - -See also: [AsyncCryptoBackend](./primitives.md#interface-asynccryptobackend) - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - --- ### Function: swapBytes32 @@ -5514,7 +5422,7 @@ console.log(toBase64(bytes)); // Outputs: SGVsbG8= ``` ```ts -export function toBase64(byteArray: number[] | Uint8Array): string +export function toBase64(byteArray: number[]): string ``` Returns @@ -5528,33 +5436,6 @@ Argument Details Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) ---- -### Function: unregisterAsyncCryptoBackend - -Removes `backend` if it is still the active optional implementation. - -```ts -export function unregisterAsyncCryptoBackend(backend: AsyncCryptoBackend): void -``` - -See also: [AsyncCryptoBackend](./primitives.md#interface-asynccryptobackend) - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- -### Function: validateAsyncCryptoBytes - -Reject malformed output from an optional cryptography backend before it can -be interpreted as key or signature material. - -```ts -export function validateAsyncCryptoBytes(operation: AsyncCryptoOperation, value: Uint8Array, expectedLength?: number): Uint8Array -``` - -See also: [AsyncCryptoOperation](./primitives.md#type-asynccryptooperation) - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - --- ### Function: verifyNotNull @@ -5590,27 +5471,6 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( --- ## Types -| | -| --- | -| [AsyncCryptoOperation](#type-asynccryptooperation) | -| [P256Point](#type-p256point) | - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- - -### Type: AsyncCryptoOperation - -Generic operations that an optional asynchronous cryptography backend can -accelerate without changing the SDK's synchronous primitive APIs. - -```ts -export type AsyncCryptoOperation = "signDigest" | "verifyDigest" | "verifyDigestBatch" | "publicKeyFromPrivate" | "multiplyPublicKey" | "tweakPublicKeyAdd" | "tweakPrivateKeyAdd" -``` - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- ### Type: P256Point ```ts @@ -5629,25 +5489,25 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | | | | | --- | --- | --- | -| [BI_EIGHT](#variable-bi_eight) | [biModSub](#variable-bimodsub) | [multiply](#variable-multiply) | -| [BI_FOUR](#variable-bi_four) | [checkBit](#variable-checkbit) | [rightShift](#variable-rightshift) | -| [BI_ONE](#variable-bi_one) | [encode](#variable-encode) | [ripemd160](#variable-ripemd160) | -| [BI_THREE](#variable-bi_three) | [exclusiveOR](#variable-exclusiveor) | [scalarMultiplyWNAF](#variable-scalarmultiplywnaf) | -| [BI_TWO](#variable-bi_two) | [fromBase58](#variable-frombase58) | [sha1](#variable-sha1) | -| [BI_ZERO](#variable-bi_zero) | [fromBase58Check](#variable-frombase58check) | [sha256](#variable-sha256) | -| [GX_BIGINT](#variable-gx_bigint) | [getBytes](#variable-getbytes) | [sha256hmac](#variable-sha256hmac) | -| [GY_BIGINT](#variable-gy_bigint) | [getBytes64](#variable-getbytes64) | [sha512](#variable-sha512) | -| [MASK_256](#variable-mask_256) | [hash160](#variable-hash160) | [sha512hmac](#variable-sha512hmac) | -| [N_BIGINT](#variable-n_bigint) | [hash256](#variable-hash256) | [sign](#variable-sign) | -| [P_BIGINT](#variable-p_bigint) | [hexToUint8Array](#variable-hextouint8array) | [toArray](#variable-toarray) | -| [P_PLUS1_DIV4](#variable-p_plus1_div4) | [incrementLeastSignificantThirtyTwoBits](#variable-incrementleastsignificantthirtytwobits) | [toBase58](#variable-tobase58) | -| [biMod](#variable-bimod) | [jpAdd](#variable-jpadd) | [toBase58Check](#variable-tobase58check) | -| [biModAdd](#variable-bimodadd) | [jpDouble](#variable-jpdouble) | [toHex](#variable-tohex) | -| [biModInv](#variable-bimodinv) | [jpNeg](#variable-jpneg) | [toSafeString](#variable-tosafestring) | -| [biModMul](#variable-bimodmul) | [minimallyEncode](#variable-minimallyencode) | [toUTF8](#variable-toutf8) | -| [biModPow](#variable-bimodpow) | [modInvN](#variable-modinvn) | [toUint8Array](#variable-touint8array) | -| [biModSqr](#variable-bimodsqr) | [modMulN](#variable-modmuln) | [verify](#variable-verify) | -| [biModSqrt](#variable-bimodsqrt) | [modN](#variable-modn) | [zero2](#variable-zero2) | +| [BI_EIGHT](#variable-bi_eight) | [biModSub](#variable-bimodsub) | [rightShift](#variable-rightshift) | +| [BI_FOUR](#variable-bi_four) | [checkBit](#variable-checkbit) | [ripemd160](#variable-ripemd160) | +| [BI_ONE](#variable-bi_one) | [encode](#variable-encode) | [scalarMultiplyWNAF](#variable-scalarmultiplywnaf) | +| [BI_THREE](#variable-bi_three) | [exclusiveOR](#variable-exclusiveor) | [sha1](#variable-sha1) | +| [BI_TWO](#variable-bi_two) | [fromBase58](#variable-frombase58) | [sha256](#variable-sha256) | +| [BI_ZERO](#variable-bi_zero) | [fromBase58Check](#variable-frombase58check) | [sha256hmac](#variable-sha256hmac) | +| [GX_BIGINT](#variable-gx_bigint) | [getBytes](#variable-getbytes) | [sha512](#variable-sha512) | +| [GY_BIGINT](#variable-gy_bigint) | [getBytes64](#variable-getbytes64) | [sha512hmac](#variable-sha512hmac) | +| [MASK_256](#variable-mask_256) | [hash160](#variable-hash160) | [sign](#variable-sign) | +| [N_BIGINT](#variable-n_bigint) | [hash256](#variable-hash256) | [toArray](#variable-toarray) | +| [P_BIGINT](#variable-p_bigint) | [incrementLeastSignificantThirtyTwoBits](#variable-incrementleastsignificantthirtytwobits) | [toBase58](#variable-tobase58) | +| [P_PLUS1_DIV4](#variable-p_plus1_div4) | [jpAdd](#variable-jpadd) | [toBase58Check](#variable-tobase58check) | +| [biMod](#variable-bimod) | [jpDouble](#variable-jpdouble) | [toHex](#variable-tohex) | +| [biModAdd](#variable-bimodadd) | [jpNeg](#variable-jpneg) | [toUTF8](#variable-toutf8) | +| [biModInv](#variable-bimodinv) | [minimallyEncode](#variable-minimallyencode) | [toUint8Array](#variable-touint8array) | +| [biModMul](#variable-bimodmul) | [modInvN](#variable-modinvn) | [verify](#variable-verify) | +| [biModPow](#variable-bimodpow) | [modMulN](#variable-modmuln) | [zero2](#variable-zero2) | +| [biModSqr](#variable-bimodsqr) | [modN](#variable-modn) | | +| [biModSqrt](#variable-bimodsqrt) | [multiply](#variable-multiply) | | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -5766,7 +5626,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Variable: biMod ```ts -biMod = (a: bigint): bigint => red(((a % P_BIGINT) + P_BIGINT) % P_BIGINT) +biMod = (a: bigint): bigint => red((a % P_BIGINT + P_BIGINT) % P_BIGINT) ``` See also: [P_BIGINT](./primitives.md#variable-p_bigint), [red](./primitives.md#function-red) @@ -5883,7 +5743,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ```ts checkBit = function (byteArray: number[], byteIndex: number, bitIndex: number): 1 | 0 { - return (byteArray[byteIndex] & (1 << bitIndex)) === 0 ? 0 : 1; + return (byteArray[byteIndex] & (1 << bitIndex)) !== 0 ? 1 : 0; } ``` @@ -5933,19 +5793,19 @@ fromBase58 = (str: string): number[] => { if (str === "" || typeof str !== "string") { throw new Error(`Expected base58 string but got “${str}”`); } - const match: string[] | null = str.match(/[^1-9A-HJ-NP-Za-km-z]/gmu); + const match: string[] | null = str.match(/[IOl0]/gmu); if (match !== null) { throw new Error(`Invalid base58 character “${match.join("")}”`); } const lz = str.match(/^1+/gmu); - const psz: number = lz === null ? 0 : lz[0].length; + const psz: number = (lz !== null) ? lz[0].length : 0; const size = ((str.length - psz) * (Math.log(58) / Math.log(256)) + 1) >>> 0; const uint8 = new Uint8Array([ ...new Uint8Array(psz), - ...Array.from(str) - .map(i => base58chars.indexOf(i)) + ...(str.match(/./gmu) ?? []) + .map((i) => base58chars.indexOf(i)) .reduce((acc, i) => { - acc = acc.map(j => { + acc = acc.map((j) => { const x = j * 58 + i; i = x >> 8; return x; @@ -5953,7 +5813,7 @@ fromBase58 = (str: string): number[] => { return acc; }, new Uint8Array(size)) .reverse() - .filter((lastValue => value => (lastValue = lastValue || value))(false)) + .filter(((lastValue) => (value) => (lastValue = lastValue || value))(false)) ]); return [...uint8]; } @@ -6036,16 +5896,13 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Variable: hash160 ```ts -hash160 = (msg: HashInput, enc?: "hex" | "utf8"): number[] => { - const first = sha256Bytes(msg, enc); - const native = ripemd160Bytes(first); - if (native != null) - return Array.from(native); +hash160 = (msg: Uint8Array | number[] | string, enc?: "hex" | "utf8"): number[] => { + const first = new SHA256().update(msg, enc).digest(); return new RIPEMD160().update(first).digest(); } ``` -See also: [RIPEMD160](./primitives.md#class-ripemd160) +See also: [RIPEMD160](./primitives.md#class-ripemd160), [SHA256](./primitives.md#class-sha256) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -6053,36 +5910,13 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Variable: hash256 ```ts -hash256 = (msg: HashInput, enc?: "hex" | "utf8"): number[] => { - return Array.from(sha256Bytes(sha256Bytes(msg, enc))); -} -``` - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- -### Variable: hexToUint8Array - -```ts -hexToUint8Array = (msg: string): Uint8Array => { - assertValidHex(msg); - const normalized = msg.length % 2 === 0 ? msg : "0" + msg; - if (CAN_USE_BUFFER) { - const decoded = BufferCtor.from(normalized, "hex"); - return new Uint8Array(decoded.buffer, decoded.byteOffset, decoded.byteLength); - } - const out = new Uint8Array(normalized.length / 2); - let o = 0; - for (let i = 0; i < normalized.length; i += 2) { - const hi = HEX_CHAR_TO_VALUE[normalized.codePointAt(i) as number]; - const lo = HEX_CHAR_TO_VALUE[normalized.codePointAt(i + 1) as number]; - out[o++] = (hi << 4) | lo; - } - return out; +hash256 = (msg: Uint8Array | number[] | string, enc?: "hex" | "utf8"): number[] => { + const first = new SHA256().update(msg, enc).digest(); + return new SHA256().update(first).digest(); } ``` -See also: [assertValidHex](./primitives.md#function-assertvalidhex) +See also: [SHA256](./primitives.md#class-sha256) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -6185,26 +6019,26 @@ minimallyEncode = (buf: number[]): number[] => { if (buf.length === 0) { return buf; } - const last = buf.at(-1)!; + const last = buf[buf.length - 1]; if ((last & 127) !== 0) { return buf; } if (buf.length === 1) { return []; } - if ((buf.at(-2)! & 128) !== 0) { + if ((buf[buf.length - 2] & 128) !== 0) { return buf; } for (let i = buf.length - 1; i > 0; i--) { if (buf[i - 1] !== 0) { - if ((buf[i - 1] & 128) === 0) { - buf[i - 1]! |= last; - return buf.slice(0, i); - } - else { + if ((buf[i - 1] & 128) !== 0) { buf[i] = last; return buf.slice(0, i + 1); } + else { + buf[i - 1] |= last; + return buf.slice(0, i); + } } } return []; @@ -6319,9 +6153,6 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ```ts ripemd160 = (msg: number[] | string, enc?: "hex" | "utf8"): number[] => { - const native = ripemd160Bytes(msg, enc); - if (native != null) - return Array.from(native); return new RIPEMD160().update(msg, enc).digest(); } ``` @@ -6338,15 +6169,48 @@ scalarMultiplyWNAF = (k: bigint, P0: { x: bigint; y: bigint; }, window: number = 5): JacobianPointBI => { - const table = wnafTable(window, P0); - const wnaf = wnafDigits(k, window); + const key = `${window}:${P0.x.toString(16)}:${P0.y.toString(16)}`; + let tbl = WNAF_TABLE_CACHE.get(key); + let P: JacobianPointBI; + if (tbl === undefined) { + const tblSize = 1 << (window - 1); + tbl = new Array(tblSize); + P = { X: P0.x, Y: P0.y, Z: BI_ONE }; + tbl[0] = P; + const twoP = jpDouble(P); + for (let i = 1; i < tblSize; i++) { + tbl[i] = jpAdd(tbl[i - 1], twoP); + } + WNAF_TABLE_CACHE.set(key, tbl); + } + else { + P = tbl[0]; + } + const wnaf: number[] = []; + const wBig = 1n << BigInt(window); + const wHalf = wBig >> 1n; + let kTmp = k; + while (kTmp > 0n) { + if ((kTmp & BI_ONE) === BI_ZERO) { + wnaf.push(0); + kTmp >>= BI_ONE; + } + else { + let z = kTmp & (wBig - 1n); + if (z > wHalf) + z -= wBig; + wnaf.push(Number(z)); + kTmp -= z; + kTmp >>= BI_ONE; + } + } let Q: JacobianPointBI = { X: BI_ZERO, Y: BI_ONE, Z: BI_ZERO }; for (let i = wnaf.length - 1; i >= 0; i--) { Q = jpDouble(Q); const di = wnaf[i]; if (di !== 0) { const idx = Math.abs(di) >> 1; - const addend = di > 0 ? table[idx] : jpNeg(table[idx]); + const addend = di > 0 ? tbl[idx] : jpNeg(tbl[idx]); Q = jpAdd(Q, addend); } } @@ -6375,21 +6239,20 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Variable: sha256 ```ts -sha256 = (msg: HashInput, enc?: "hex" | "utf8"): number[] => { - return Array.from(sha256Bytes(msg, enc)); +sha256 = (msg: Uint8Array | number[] | string, enc?: "hex" | "utf8"): number[] => { + return new SHA256().update(msg, enc).digest(); } ``` +See also: [SHA256](./primitives.md#class-sha256) + Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- ### Variable: sha256hmac ```ts -sha256hmac = (key: HashInput, msg: HashInput, enc?: "hex"): number[] => { - const native = digestWithNodeHmac("sha256", key, msg, enc); - if (native != null) - return Array.from(native); +sha256hmac = (key: Uint8Array | number[] | string, msg: Uint8Array | number[] | string, enc?: "hex"): number[] => { return new SHA256HMAC(key).update(msg, enc).digest(); } ``` @@ -6402,21 +6265,20 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Variable: sha512 ```ts -sha512 = (msg: HashInput, enc?: "hex" | "utf8"): number[] => { - return Array.from(sha512Bytes(msg, enc)); +sha512 = (msg: number[] | string, enc?: "hex" | "utf8"): number[] => { + return new SHA512().update(msg, enc).digest(); } ``` +See also: [SHA512](./primitives.md#class-sha512) + Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- ### Variable: sha512hmac ```ts -sha512hmac = (key: HashInput, msg: HashInput, enc?: "hex"): number[] => { - const native = digestWithNodeHmac("sha512", key, msg, enc); - if (native != null) - return Array.from(native); +sha512hmac = (key: Uint8Array | number[] | string, msg: Uint8Array | number[] | string, enc?: "hex"): number[] => { return new SHA512HMAC(key).update(msg, enc).digest(); } ``` @@ -6440,16 +6302,59 @@ sign = (msg: BigNumber, key: BigNumber, forceLowS: boolean = false, customK?: Bi const bkey = key.toArray("be", bytes); const nonce = msg.toArray("be", bytes); const drbg = new DRBG(bkey, nonce); - const fixedK = BigNumber.isBN(customK); for (let iter = 0;; iter++) { - const signature = signatureFromK(selectK(customK, iter, drbg), msgBig, keyBig, forceLowS, fixedK); - if (signature != null) - return signature; + let kBN = typeof customK === "function" + ? customK(iter) + : BigNumber.isBN(customK) + ? customK + : new BigNumber(drbg.generate(bytes), 16); + if (kBN == null) { + throw new Error("k is undefined"); + } + kBN = truncateToN(kBN, true); + if (kBN.cmpn(1) < 0 || kBN.cmp(ns1) > 0) { + if (BigNumber.isBN(customK)) { + throw new Error("Invalid fixed custom K value (must be >1 and halfN) { + sBig = N_BIGINT - sBig; + } + const r = new BigNumber(rBig.toString(16), 16); + const s = new BigNumber(sBig.toString(16), 16); + return new Signature(r, s); } } ``` -See also: [BigNumber](./primitives.md#class-bignumber), [DRBG](./primitives.md#class-drbg), [Signature](./primitives.md#class-signature), [toArray](./primitives.md#variable-toarray) +See also: [BigNumber](./primitives.md#class-bignumber), [DRBG](./primitives.md#class-drbg), [N_BIGINT](./primitives.md#variable-n_bigint), [Signature](./primitives.md#class-signature), [modInvN](./primitives.md#variable-modinvn), [modMulN](./primitives.md#variable-modmuln), [modN](./primitives.md#variable-modn), [toArray](./primitives.md#variable-toarray) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -6463,7 +6368,7 @@ toArray = (msg: any, enc?: "hex" | "utf8" | "base64"): any[] => { if (msg === undefined) return []; if (typeof msg !== "string") { - return Array.from(msg, (item: any) => Math.trunc(item)); + return Array.from(msg, (item: any) => item | 0); } switch (enc) { case "hex": @@ -6485,35 +6390,31 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ```ts toBase58 = (bin: number[]): string => { - const base58Map = Array.from({ length: 256 }, () => -1); + const base58Map = Array(256).fill(-1); for (let i = 0; i < base58chars.length; ++i) { - base58Map[base58chars.codePointAt(i) as number] = i; + base58Map[base58chars.charCodeAt(i)] = i; } const result: number[] = []; for (const byte of bin) { let carry = byte; for (let j = 0; j < result.length; ++j) { const x = (base58Map[result[j]] << 8) + carry; - const quotient = Math.trunc(x / 58); - const remainder = x - quotient * 58; - result[j] = base58chars.codePointAt(remainder) as number; - carry = quotient; + result[j] = base58chars.charCodeAt(x % 58); + carry = (x / 58) | 0; } while (carry !== 0) { - const quotient = Math.trunc(carry / 58); - const remainder = carry - quotient * 58; - result.push(base58chars.codePointAt(remainder) as number); - carry = quotient; + result.push(base58chars.charCodeAt(carry % 58)); + carry = (carry / 58) | 0; } } for (const byte of bin) { - if (byte === 0) - result.push("1".codePointAt(0) as number); - else + if (byte !== 0) break; + else + result.push("1".charCodeAt(0)); } result.reverse(); - return String.fromCodePoint(...result); + return String.fromCharCode(...result); } ``` @@ -6544,42 +6445,11 @@ toHex = (msg: number[] | Uint8Array): string => { } if (msg.length === 0) return ""; - return Array.from(msg, byte => HEX_BYTE_STRINGS[byte & 255]).join(""); -} -``` - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- -### Variable: toSafeString - -```ts -toSafeString = (value: unknown, fallback = "Unknown value"): string => { - if (value === null) - return "null"; - if (value === undefined) - return "undefined"; - if (typeof value === "string") - return value; - if (typeof value === "number" || typeof value === "bigint") - return value.toString(); - if (typeof value === "boolean") - return value ? "true" : "false"; - if (typeof value === "symbol") - return value.description ?? value.toString(); - if (value instanceof Error && value.message.length > 0) - return value.message; - const message = (value as { - message?: unknown; - }).message; - if (typeof message === "string" && message.length > 0) - return message; - try { - return JSON.stringify(value) ?? fallback; - } - catch { - return fallback; + const out = new Array(msg.length); + for (let i = 0; i < msg.length; i++) { + out[i] = HEX_BYTE_STRINGS[msg[i] & 255]; } + return out.join(""); } ``` @@ -6589,8 +6459,8 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Variable: toUTF8 ```ts -toUTF8 = (arr: number[] | Uint8Array): string => { - return new TextDecoder().decode(arr instanceof Uint8Array ? arr : new Uint8Array(arr)); +toUTF8 = (arr: number[]): string => { + return new TextDecoder().decode(new Uint8Array(arr)); } ``` @@ -6603,13 +6473,11 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( toUint8Array = (msg: any, enc?: "hex" | "utf8" | "base64"): Uint8Array => { if (msg instanceof Uint8Array) return msg; - if (typeof msg === "string" && enc === "hex") - return hexToUint8Array(msg); return new Uint8Array(toArray(msg, enc)); } ``` -See also: [hexToUint8Array](./primitives.md#variable-hextouint8array), [toArray](./primitives.md#variable-toarray) +See also: [toArray](./primitives.md#variable-toarray) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -6623,7 +6491,7 @@ verify = (msg: BigNumber, sig: Signature, key: Point): boolean => { return false; } const hash = bnToBigInt(msg); - if (key.x == null || key.y == null) { + if ((key.x == null) || (key.y == null)) { throw new Error("Invalid public key: missing coordinates."); } const publicKey = { diff --git a/packages/sdk/docs/reference/registry.md b/packages/sdk/docs/reference/registry.md index 606482f1b..df4ab4e91 100644 --- a/packages/sdk/docs/reference/registry.md +++ b/packages/sdk/docs/reference/registry.md @@ -216,7 +216,6 @@ export class RegistryClient { constructor(private readonly wallet: WalletInterface = new WalletClient(), options: { acceptDelayedBroadcast?: boolean; resolver?: LookupResolver; - networkPreset?: LookupNetworkPreset; } = {}, private readonly originator?: OriginatorDomainNameStringUnder250Bytes) async registerDefinition(data: DefinitionData): Promise async resolve(definitionType: T, query: RegistryQueryMapping[T]): Promise @@ -226,7 +225,7 @@ export class RegistryClient { } ``` -See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [BroadcastResponse](./transaction.md#interface-broadcastresponse), [DefinitionData](./registry.md#type-definitiondata), [DefinitionType](./registry.md#type-definitiontype), [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset), [LookupResolver](./overlay-tools.md#class-lookupresolver), [OriginatorDomainNameStringUnder250Bytes](./wallet.md#type-originatordomainnamestringunder250bytes), [RegistryQueryMapping](./registry.md#interface-registryquerymapping), [RegistryRecord](./registry.md#type-registryrecord), [WalletClient](./wallet.md#class-walletclient), [WalletInterface](./wallet.md#interface-walletinterface) +See also: [BroadcastFailure](./transaction.md#interface-broadcastfailure), [BroadcastResponse](./transaction.md#interface-broadcastresponse), [DefinitionData](./registry.md#type-definitiondata), [DefinitionType](./registry.md#type-definitiontype), [LookupResolver](./overlay-tools.md#class-lookupresolver), [OriginatorDomainNameStringUnder250Bytes](./wallet.md#type-originatordomainnamestringunder250bytes), [RegistryQueryMapping](./registry.md#interface-registryquerymapping), [RegistryRecord](./registry.md#type-registryrecord), [WalletClient](./wallet.md#class-walletclient), [WalletInterface](./wallet.md#interface-walletinterface) #### Method listOwnRegistryEntries diff --git a/packages/sdk/docs/reference/remittance.md b/packages/sdk/docs/reference/remittance.md index 82cc387d1..d27ef18dc 100644 --- a/packages/sdk/docs/reference/remittance.md +++ b/packages/sdk/docs/reference/remittance.md @@ -191,6 +191,16 @@ Description applied to created actions. description?: string ``` +#### Property internalizeProtocol + +Deprecated. BRC-29 settlements must be internalized as `wallet payment` so the +recipient can verify and manage the derived output. The module rejects +`basket insertion`; it is reserved for application-managed custom outputs. + +```ts +internalizeProtocol?: "wallet payment" | "basket insertion" +``` + #### Property labels Labels applied to created actions. diff --git a/packages/sdk/docs/reference/script.md b/packages/sdk/docs/reference/script.md index b7943846f..b3c065526 100644 --- a/packages/sdk/docs/reference/script.md +++ b/packages/sdk/docs/reference/script.md @@ -29,8 +29,7 @@ export interface FormatPreimageParams { sourceTXID: string; sourceSatoshis: number; lockingScript: Script; - otherInputs?: Transaction["inputs"]; - allInputs?: Transaction["inputs"]; + otherInputs: Transaction["inputs"]; inputSequence?: number; } ``` @@ -48,7 +47,6 @@ A representation of a chunk of a script, which includes an opcode. For push oper export default interface ScriptChunk { op: number; data?: number[]; - invalidLength?: boolean; } ``` @@ -107,12 +105,8 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( --- ### Interface: SpendVerificationContext -Explicit chain context for script verification. - -Transaction version is script data, not a reliable signal for whether a -caller is asking for consensus or policy validation. Backends should use -this context when it is supplied and retain their compatibility behavior -only when it is omitted. +Explicit chain context for script verification. Transaction version is script +data and is not a policy/consensus selector. ```ts export default interface SpendVerificationContext { @@ -123,75 +117,25 @@ export default interface SpendVerificationContext { } ``` -#### Property blockHeight - -Height of the block against which the spend is evaluated. - -```ts -blockHeight?: number -``` - -#### Property consensus - -`true` selects consensus rules; `false` permits policy validation. - -```ts -consensus: boolean -``` - -#### Property utxoHeight - -Height at which the source output was mined. - -```ts -utxoHeight?: number -``` - -#### Property verifyFlags - -Optional backend-specific script verification flags. - -```ts -verifyFlags?: string | string[] -``` - Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- ### Interface: SpendVerifierInterface +An asynchronous backend capable of validating a single Spend-shaped input. +The explicit context is authoritative when supplied. + ```ts export default interface SpendVerifierInterface { isReady?: () => boolean; shouldVerifySpend?: (spend: Spend, context?: SpendVerificationContext) => boolean; verifySpend: (spend: Spend, context?: SpendVerificationContext) => Promise; - verifySpendsBatch?: (items: ReadonlyArray & { - spend: Spend; - }>) => Promise; verifySpendSync?: (spend: Spend, context?: SpendVerificationContext) => boolean; } ``` See also: [Spend](./script.md#class-spend), [SpendVerificationContext](./script.md#interface-spendverificationcontext) -#### Property isReady - -Optional synchronous readiness signal for compatibility APIs. - -```ts -isReady?: () => boolean -``` - -#### Property shouldVerifySpend - -Optionally decide whether this backend should handle the Spend now. -Returning false preserves the existing synchronous JavaScript validator. - -```ts -shouldVerifySpend?: (spend: Spend, context?: SpendVerificationContext) => boolean -``` -See also: [Spend](./script.md#class-spend), [SpendVerificationContext](./script.md#interface-spendverificationcontext) - Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- @@ -222,8 +166,8 @@ Inherits all properties and methods from the Script class. ```ts export default class LockingScript extends Script { - override isLockingScript(): boolean - override isUnlockingScript(): boolean + isLockingScript(): boolean + isUnlockingScript(): boolean } ``` @@ -232,7 +176,7 @@ See also: [Script](./script.md#class-script) #### Method isLockingScript ```ts -override isLockingScript(): boolean +isLockingScript(): boolean ``` Returns @@ -242,7 +186,7 @@ Always returns true for a LockingScript instance. #### Method isUnlockingScript ```ts -override isUnlockingScript(): boolean +isUnlockingScript(): boolean ``` Returns @@ -541,8 +485,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( export default class Script { static fromASM(asm: string): Script static fromHex(hex: string): Script - static fromBinary(bin: number[] | Uint8Array): Script - static fromBinaryView(bin: Uint8Array): Script + static fromBinary(bin: number[]): Script constructor(chunks: ScriptChunk[] = [], rawBytesCache?: Uint8Array, hexCache?: string, parsed: boolean = true) get chunks(): ScriptChunk[] set chunks(value: ScriptChunk[]) @@ -550,14 +493,14 @@ export default class Script { toHex(): string toBinary(): number[] toUint8Array(): Uint8Array - writeScript(script: Script): this - writeOpCode(op: number): this - setChunkOpCode(i: number, op: number): this - writeBn(bn: BigNumber): this - writeBin(bin: number[]): this - writeNumber(num: number): this - removeCodeseparators(): this - findAndDelete(script: Script): this + writeScript(script: Script): Script + writeOpCode(op: number): Script + setChunkOpCode(i: number, op: number): Script + writeBn(bn: BigNumber): Script + writeBin(bin: number[]): Script + writeNumber(num: number): Script + removeCodeseparators(): Script + findAndDelete(script: Script): Script isPushOnly(): boolean isLockingScript(): boolean isUnlockingScript(): boolean @@ -589,7 +532,7 @@ Argument Details Deletes the given item wherever it appears in the current script. ```ts -findAndDelete(script: Script): this +findAndDelete(script: Script): Script ``` See also: [Script](./script.md#class-script) @@ -627,7 +570,7 @@ const script = Script.fromASM("OP_DUP OP_HASH160 abcd... OP_EQUALVERIFY OP_CHECK #### Method fromBinary ```ts -static fromBinary(bin: number[] | Uint8Array): Script +static fromBinary(bin: number[]): Script ``` See also: [Script](./script.md#class-script) @@ -646,16 +589,6 @@ Example const script = Script.fromBinary([0x76, 0xa9, ...]) ``` -#### Method fromBinaryView - -Constructs a lazily parsed script over an existing byte view without a copy. -The caller must not mutate `bin` while the script is in use. - -```ts -static fromBinaryView(bin: Uint8Array): Script -``` -See also: [Script](./script.md#class-script) - #### Method fromHex ```ts @@ -711,8 +644,9 @@ True if the script is an unlocking script, otherwise false. #### Method removeCodeseparators ```ts -removeCodeseparators(): this +removeCodeseparators(): Script ``` +See also: [Script](./script.md#class-script) Returns @@ -721,8 +655,9 @@ This script instance for chaining. #### Method setChunkOpCode ```ts -setChunkOpCode(i: number, op: number): this +setChunkOpCode(i: number, op: number): Script ``` +See also: [Script](./script.md#class-script) Returns @@ -768,8 +703,9 @@ The script in hexadecimal format. #### Method writeBin ```ts -writeBin(bin: number[]): this +writeBin(bin: number[]): Script ``` +See also: [Script](./script.md#class-script) Returns @@ -787,9 +723,9 @@ Throws an error if the data is too large to be pushed. #### Method writeBn ```ts -writeBn(bn: BigNumber): this +writeBn(bn: BigNumber): Script ``` -See also: [BigNumber](./primitives.md#class-bignumber) +See also: [BigNumber](./primitives.md#class-bignumber), [Script](./script.md#class-script) Returns @@ -803,8 +739,9 @@ Argument Details #### Method writeNumber ```ts -writeNumber(num: number): this +writeNumber(num: number): Script ``` +See also: [Script](./script.md#class-script) Returns @@ -818,8 +755,9 @@ Argument Details #### Method writeOpCode ```ts -writeOpCode(op: number): this +writeOpCode(op: number): Script ``` +See also: [Script](./script.md#class-script) Returns @@ -833,7 +771,7 @@ Argument Details #### Method writeScript ```ts -writeScript(script: Script): this +writeScript(script: Script): Script ``` See also: [Script](./script.md#class-script) @@ -882,19 +820,19 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( --- ### Class: ScriptResourceLimitError -Raised when an explicitly configured local interpreter budget is exhausted. - -This is deliberately distinct from ScriptEvaluationError: exhausting a -caller-supplied resource budget does not prove that a script is invalid. +Raised when a caller-supplied local interpreter budget or a host representation +limit is exhausted. This is deliberately distinct from +`ScriptEvaluationError`: resource exhaustion does not prove script invalidity. ```ts export default class ScriptResourceLimitError extends Error { - constructor(public readonly resource: ScriptResource, public readonly limit: number | bigint, public readonly attempted: number | bigint) + readonly resource: ScriptResource; + readonly limit: number | bigint; + readonly attempted: number | bigint; + constructor(resource: ScriptResource, limit: number | bigint, attempted: number | bigint) } ``` -See also: [ScriptResource](./script.md#type-scriptresource) - Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- @@ -912,27 +850,22 @@ export default class Spend { lockingScript: LockingScript; transactionVersion: number; otherInputs: TransactionInput[]; - allInputs?: TransactionInput[]; outputs: TransactionOutput[]; inputIndex: number; unlockingScript: UnlockingScript; inputSequence: number; lockTime: number; - context!: "UnlockingScript" | "LockingScript"; - programCounter!: number; - lastCodeSeparator!: number | null; + context: "UnlockingScript" | "LockingScript"; + programCounter: number; + lastCodeSeparator: number | null; stack: number[][]; altStack: number[][]; ifStack: boolean[]; - elseStack: boolean[]; memoryLimit: number; readonly hasExplicitMemoryLimit: boolean; stackMem: number; altStackMem: number; isRelaxedOverride: boolean; - verifyFlags?: Set; - executedOpCount: number; - returningFromConditional: boolean; constructor(params: { sourceTXID: string; sourceOutputIndex: number; @@ -940,7 +873,6 @@ export default class Spend { lockingScript: LockingScript; transactionVersion: number; otherInputs: TransactionInput[]; - allInputs?: TransactionInput[]; outputs: TransactionOutput[]; unlockingScript: UnlockingScript; inputSequence: number; @@ -948,19 +880,16 @@ export default class Spend { lockTime: number; memoryLimit?: number; isRelaxed?: boolean; - verifyFlags?: string | string[]; - sigHashCache?: SignatureHashCache; }) reset(): void step(): boolean - validate(context?: SpendVerificationContext): boolean - validateJavaScript(): boolean - async validateWith(verifier: SpendVerifierInterface, context?: SpendVerificationContext): Promise - toTransactionUint8Array(): Uint8Array + validate(context?: SpendVerificationContext): boolean + validateWith(verifier: SpendVerifierInterface, context?: SpendVerificationContext): Promise + toTransactionUint8Array(): Uint8Array } ``` -See also: [LockingScript](./script.md#class-lockingscript), [SignatureHashCache](./primitives.md#interface-signaturehashcache), [SpendVerificationContext](./script.md#interface-spendverificationcontext), [SpendVerifierInterface](./script.md#interface-spendverifierinterface), [TransactionInput](./transaction.md#interface-transactioninput), [TransactionOutput](./transaction.md#interface-transactionoutput), [UnlockingScript](./script.md#class-unlockingscript) +See also: [LockingScript](./script.md#class-lockingscript), [SpendVerificationContext](./script.md#interface-spendverificationcontext), [SpendVerifierInterface](./script.md#interface-spendverifierinterface), [TransactionInput](./transaction.md#interface-transactioninput), [TransactionOutput](./transaction.md#interface-transactionoutput), [UnlockingScript](./script.md#class-unlockingscript) #### Constructor @@ -972,7 +901,6 @@ constructor(params: { lockingScript: LockingScript; transactionVersion: number; otherInputs: TransactionInput[]; - allInputs?: TransactionInput[]; outputs: TransactionOutput[]; unlockingScript: UnlockingScript; inputSequence: number; @@ -980,11 +908,9 @@ constructor(params: { lockTime: number; memoryLimit?: number; isRelaxed?: boolean; - verifyFlags?: string | string[]; - sigHashCache?: SignatureHashCache; }) ``` -See also: [LockingScript](./script.md#class-lockingscript), [SignatureHashCache](./primitives.md#interface-signaturehashcache), [TransactionInput](./transaction.md#interface-transactioninput), [TransactionOutput](./transaction.md#interface-transactionoutput), [UnlockingScript](./script.md#class-unlockingscript) +See also: [LockingScript](./script.md#class-lockingscript), [TransactionInput](./transaction.md#interface-transactioninput), [TransactionOutput](./transaction.md#interface-transactionoutput), [UnlockingScript](./script.md#class-unlockingscript) Argument Details @@ -1013,9 +939,8 @@ The outputs of the current transaction. + **params.lockTime** + The lock time of the transaction. + **params.memoryLimit** - + Optional caller-supplied local -interpreter budget. Resource exhaustion is reported separately from -script invalidity. + + Optional caller-supplied local interpreter budget. If omitted, +post-Genesis validation has no arbitrary SDK memory cap. + **params.isRelaxed** + Optional. If true, disables all the unlocking script maleability restrictions consitent with Chronicle release. Maleability restrictions are neve appliced to locking scripts. @@ -1037,70 +962,48 @@ const spend = new Spend({ }); ``` -#### Method toTransactionUint8Array - -Serializes the ordinary transaction represented by this Spend. The source -output is intentionally excluded and is supplied separately to a Spend -verifier, avoiding an EF construction and parse for one-input validation. - -```ts -toTransactionUint8Array(): Uint8Array -``` - #### Method validate ```ts -validate(context?: SpendVerificationContext): boolean +validate(context?: SpendVerificationContext): boolean ``` + See also: [SpendVerificationContext](./script.md#interface-spendverificationcontext) Returns -Returns true when the spend is valid. - -Argument Details - -+ **context** - + Optional explicit consensus or -policy context passed to a registered script backend. - -Throws - -If script validation fails. - -If a local interpreter resource is -exhausted before validity can be determined. +Returns true if the scripts are valid and the spend is legitimate, otherwise false. Example ```ts -spend.validate() -console.log("Spend is valid!") +if (spend.validate()) { + console.log("Spend is valid!"); +} else { + console.log("Invalid spend!"); +} ``` -#### Method validateJavaScript +#### Method validateWith -Runs the original TypeScript interpreter explicitly, bypassing any -registered optional backend. +Validates this spend with an asynchronous pluggable backend. Backend errors are +propagated without silently falling back to the JavaScript interpreter. ```ts -validateJavaScript(): boolean +validateWith(verifier: SpendVerifierInterface, context?: SpendVerificationContext): Promise ``` -#### Method validateWith - -```ts -async validateWith(verifier: SpendVerifierInterface, context?: SpendVerificationContext): Promise -``` See also: [SpendVerificationContext](./script.md#interface-spendverificationcontext), [SpendVerifierInterface](./script.md#interface-spendverifierinterface) -Argument Details +#### Method toTransactionUint8Array + +Serializes the ordinary transaction represented by this Spend. The source +output is supplied separately to a Spend verifier, avoiding EF construction +for one-input validation. -+ **verifier** - + The backend used when it accepts this Spend. -+ **context** - + Optional explicit consensus or policy context. Transaction -version is never used as a substitute for this context. +```ts +toTransactionUint8Array(): Uint8Array +``` Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -1114,8 +1017,8 @@ Inherits all properties and methods from the Script class. ```ts export default class UnlockingScript extends Script { - override isLockingScript(): boolean - override isUnlockingScript(): boolean + isLockingScript(): boolean + isUnlockingScript(): boolean } ``` @@ -1124,7 +1027,7 @@ See also: [Script](./script.md#class-script) #### Method isLockingScript ```ts -override isLockingScript(): boolean +isLockingScript(): boolean ``` Returns @@ -1134,7 +1037,7 @@ Always returns false for an UnlockingScript instance. #### Method isUnlockingScript ```ts -override isUnlockingScript(): boolean +isUnlockingScript(): boolean ``` Returns @@ -1191,7 +1094,6 @@ export function resolveSourceDetails(tx: Transaction, inputIndex: number, provid sourceSatoshis: number; lockingScript: Script; otherInputs: typeof tx.inputs; - allInputs: typeof tx.inputs; } ``` @@ -1208,10 +1110,6 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( export type ScriptResource = "stack" | "alt-stack" | "element-size" ``` -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- ## Enums ## Variables - diff --git a/packages/sdk/docs/reference/storage.md b/packages/sdk/docs/reference/storage.md index c6407b2a6..c1579f7a1 100644 --- a/packages/sdk/docs/reference/storage.md +++ b/packages/sdk/docs/reference/storage.md @@ -37,12 +37,10 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ```ts export interface DownloaderConfig { - networkPreset: LookupNetworkPreset; + networkPreset: "mainnet" | "testnet" | "teratestnet" | "local"; } ``` -See also: [LookupNetworkPreset](./overlay-tools.md#type-lookupnetworkpreset) - Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- diff --git a/packages/sdk/docs/reference/wallet.md b/packages/sdk/docs/reference/wallet.md index 2c2140882..067a1ed89 100644 --- a/packages/sdk/docs/reference/wallet.md +++ b/packages/sdk/docs/reference/wallet.md @@ -6,40 +6,40 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | | | | | --- | --- | --- | -| [AbortActionArgs](#interface-abortactionargs) | [ListCertificatesArgs](#interface-listcertificatesargs) | [ValidInternalizeActionArgs](#interface-validinternalizeactionargs) | -| [AbortActionResult](#interface-abortactionresult) | [ListCertificatesResult](#interface-listcertificatesresult) | [ValidInternalizeOutput](#interface-validinternalizeoutput) | -| [AcquireCertificateArgs](#interface-acquirecertificateargs) | [ListOutputsArgs](#interface-listoutputsargs) | [ValidListActionsArgs](#interface-validlistactionsargs) | -| [AuthenticatedResult](#interface-authenticatedresult) | [ListOutputsResult](#interface-listoutputsresult) | [ValidListCertificatesArgs](#interface-validlistcertificatesargs) | -| [BasketInsertion](#interface-basketinsertion) | [OutPoint](#interface-outpoint) | [ValidListOutputsArgs](#interface-validlistoutputsargs) | -| [CertificateResult](#interface-certificateresult) | [PrivateKeyDerivation](#interface-privatekeyderivation) | [ValidProcessActionArgs](#interface-validprocessactionargs) | -| [CreateActionArgs](#interface-createactionargs) | [ProveCertificateArgs](#interface-provecertificateargs) | [ValidProcessActionOptions](#interface-validprocessactionoptions) | -| [CreateActionInput](#interface-createactioninput) | [ProveCertificateResult](#interface-provecertificateresult) | [ValidProveCertificateArgs](#interface-validprovecertificateargs) | -| [CreateActionOptions](#interface-createactionoptions) | [RelinquishCertificateArgs](#interface-relinquishcertificateargs) | [ValidRelinquishCertificateArgs](#interface-validrelinquishcertificateargs) | -| [CreateActionOutput](#interface-createactionoutput) | [RelinquishCertificateResult](#interface-relinquishcertificateresult) | [ValidRelinquishOutputArgs](#interface-validrelinquishoutputargs) | -| [CreateActionResult](#interface-createactionresult) | [RelinquishOutputArgs](#interface-relinquishoutputargs) | [ValidSignActionArgs](#interface-validsignactionargs) | -| [CreateHmacArgs](#interface-createhmacargs) | [RelinquishOutputResult](#interface-relinquishoutputresult) | [ValidSignActionOptions](#interface-validsignactionoptions) | -| [CreateHmacResult](#interface-createhmacresult) | [RevealCounterpartyKeyLinkageArgs](#interface-revealcounterpartykeylinkageargs) | [ValidWalletPayment](#interface-validwalletpayment) | -| [CreateSignatureArgs](#interface-createsignatureargs) | [RevealCounterpartyKeyLinkageResult](#interface-revealcounterpartykeylinkageresult) | [ValidWalletSignerArgs](#interface-validwalletsignerargs) | -| [CreateSignatureResult](#interface-createsignatureresult) | [RevealSpecificKeyLinkageArgs](#interface-revealspecifickeylinkageargs) | [VerifyHmacArgs](#interface-verifyhmacargs) | -| [DiscoverByAttributesArgs](#interface-discoverbyattributesargs) | [RevealSpecificKeyLinkageResult](#interface-revealspecifickeylinkageresult) | [VerifyHmacResult](#interface-verifyhmacresult) | -| [DiscoverByIdentityKeyArgs](#interface-discoverbyidentitykeyargs) | [ReviewActionResult](#interface-reviewactionresult) | [VerifySignatureArgs](#interface-verifysignatureargs) | -| [DiscoverCertificatesResult](#interface-discovercertificatesresult) | [SendWithResult](#interface-sendwithresult) | [VerifySignatureResult](#interface-verifysignatureresult) | -| [GetHeaderArgs](#interface-getheaderargs) | [SignActionArgs](#interface-signactionargs) | [WalletAction](#interface-walletaction) | -| [GetHeaderResult](#interface-getheaderresult) | [SignActionOptions](#interface-signactionoptions) | [WalletActionInput](#interface-walletactioninput) | -| [GetHeightResult](#interface-getheightresult) | [SignActionResult](#interface-signactionresult) | [WalletActionOutput](#interface-walletactionoutput) | -| [GetNetworkResult](#interface-getnetworkresult) | [SignActionSpend](#interface-signactionspend) | [WalletCertificate](#interface-walletcertificate) | -| [GetPublicKeyArgs](#interface-getpublickeyargs) | [SignableTransaction](#interface-signabletransaction) | [WalletDecryptArgs](#interface-walletdecryptargs) | -| [GetPublicKeyResult](#interface-getpublickeyresult) | [ValidAbortActionArgs](#interface-validabortactionargs) | [WalletDecryptResult](#interface-walletdecryptresult) | -| [GetVersionResult](#interface-getversionresult) | [ValidAcquireCertificateArgs](#interface-validacquirecertificateargs) | [WalletEncryptArgs](#interface-walletencryptargs) | -| [IdentityCertificate](#interface-identitycertificate) | [ValidAcquireDirectCertificateArgs](#interface-validacquiredirectcertificateargs) | [WalletEncryptResult](#interface-walletencryptresult) | -| [IdentityCertifier](#interface-identitycertifier) | [ValidAcquireIssuanceCertificateArgs](#interface-validacquireissuancecertificateargs) | [WalletEncryptionArgs](#interface-walletencryptionargs) | -| [InternalizeActionArgs](#interface-internalizeactionargs) | [ValidBasketInsertion](#interface-validbasketinsertion) | [WalletErrorObject](#interface-walleterrorobject) | -| [InternalizeActionResult](#interface-internalizeactionresult) | [ValidCreateActionArgs](#interface-validcreateactionargs) | [WalletInterface](#interface-walletinterface) | -| [InternalizeOutput](#interface-internalizeoutput) | [ValidCreateActionInput](#interface-validcreateactioninput) | [WalletLoggerInterface](#interface-walletloggerinterface) | -| [KeyDeriverApi](#interface-keyderiverapi) | [ValidCreateActionOptions](#interface-validcreateactionoptions) | [WalletLoggerLog](#interface-walletloggerlog) | -| [KeyLinkageResult](#interface-keylinkageresult) | [ValidCreateActionOutput](#interface-validcreateactionoutput) | [WalletOutput](#interface-walletoutput) | -| [ListActionsArgs](#interface-listactionsargs) | [ValidDiscoverByAttributesArgs](#interface-validdiscoverbyattributesargs) | [WalletPayment](#interface-walletpayment) | -| [ListActionsResult](#interface-listactionsresult) | [ValidDiscoverByIdentityKeyArgs](#interface-validdiscoverbyidentitykeyargs) | [WalletWire](#interface-walletwire) | +| [AbortActionArgs](#interface-abortactionargs) | [ListCertificatesArgs](#interface-listcertificatesargs) | [ValidInternalizeOutput](#interface-validinternalizeoutput) | +| [AbortActionResult](#interface-abortactionresult) | [ListCertificatesResult](#interface-listcertificatesresult) | [ValidListActionsArgs](#interface-validlistactionsargs) | +| [AcquireCertificateArgs](#interface-acquirecertificateargs) | [ListOutputsArgs](#interface-listoutputsargs) | [ValidListCertificatesArgs](#interface-validlistcertificatesargs) | +| [AuthenticatedResult](#interface-authenticatedresult) | [ListOutputsResult](#interface-listoutputsresult) | [ValidListOutputsArgs](#interface-validlistoutputsargs) | +| [BasketInsertion](#interface-basketinsertion) | [OutPoint](#interface-outpoint) | [ValidProcessActionArgs](#interface-validprocessactionargs) | +| [CertificateResult](#interface-certificateresult) | [ProveCertificateArgs](#interface-provecertificateargs) | [ValidProcessActionOptions](#interface-validprocessactionoptions) | +| [CreateActionArgs](#interface-createactionargs) | [ProveCertificateResult](#interface-provecertificateresult) | [ValidProveCertificateArgs](#interface-validprovecertificateargs) | +| [CreateActionInput](#interface-createactioninput) | [RelinquishCertificateArgs](#interface-relinquishcertificateargs) | [ValidRelinquishCertificateArgs](#interface-validrelinquishcertificateargs) | +| [CreateActionOptions](#interface-createactionoptions) | [RelinquishCertificateResult](#interface-relinquishcertificateresult) | [ValidRelinquishOutputArgs](#interface-validrelinquishoutputargs) | +| [CreateActionOutput](#interface-createactionoutput) | [RelinquishOutputArgs](#interface-relinquishoutputargs) | [ValidSignActionArgs](#interface-validsignactionargs) | +| [CreateActionResult](#interface-createactionresult) | [RelinquishOutputResult](#interface-relinquishoutputresult) | [ValidSignActionOptions](#interface-validsignactionoptions) | +| [CreateHmacArgs](#interface-createhmacargs) | [RevealCounterpartyKeyLinkageArgs](#interface-revealcounterpartykeylinkageargs) | [ValidWalletPayment](#interface-validwalletpayment) | +| [CreateHmacResult](#interface-createhmacresult) | [RevealCounterpartyKeyLinkageResult](#interface-revealcounterpartykeylinkageresult) | [ValidWalletSignerArgs](#interface-validwalletsignerargs) | +| [CreateSignatureArgs](#interface-createsignatureargs) | [RevealSpecificKeyLinkageArgs](#interface-revealspecifickeylinkageargs) | [VerifyHmacArgs](#interface-verifyhmacargs) | +| [CreateSignatureResult](#interface-createsignatureresult) | [RevealSpecificKeyLinkageResult](#interface-revealspecifickeylinkageresult) | [VerifyHmacResult](#interface-verifyhmacresult) | +| [DiscoverByAttributesArgs](#interface-discoverbyattributesargs) | [ReviewActionResult](#interface-reviewactionresult) | [VerifySignatureArgs](#interface-verifysignatureargs) | +| [DiscoverByIdentityKeyArgs](#interface-discoverbyidentitykeyargs) | [SendWithResult](#interface-sendwithresult) | [VerifySignatureResult](#interface-verifysignatureresult) | +| [DiscoverCertificatesResult](#interface-discovercertificatesresult) | [SignActionArgs](#interface-signactionargs) | [WalletAction](#interface-walletaction) | +| [GetHeaderArgs](#interface-getheaderargs) | [SignActionOptions](#interface-signactionoptions) | [WalletActionInput](#interface-walletactioninput) | +| [GetHeaderResult](#interface-getheaderresult) | [SignActionResult](#interface-signactionresult) | [WalletActionOutput](#interface-walletactionoutput) | +| [GetHeightResult](#interface-getheightresult) | [SignActionSpend](#interface-signactionspend) | [WalletCertificate](#interface-walletcertificate) | +| [GetNetworkResult](#interface-getnetworkresult) | [SignableTransaction](#interface-signabletransaction) | [WalletDecryptArgs](#interface-walletdecryptargs) | +| [GetPublicKeyArgs](#interface-getpublickeyargs) | [ValidAbortActionArgs](#interface-validabortactionargs) | [WalletDecryptResult](#interface-walletdecryptresult) | +| [GetPublicKeyResult](#interface-getpublickeyresult) | [ValidAcquireCertificateArgs](#interface-validacquirecertificateargs) | [WalletEncryptArgs](#interface-walletencryptargs) | +| [GetVersionResult](#interface-getversionresult) | [ValidAcquireDirectCertificateArgs](#interface-validacquiredirectcertificateargs) | [WalletEncryptResult](#interface-walletencryptresult) | +| [IdentityCertificate](#interface-identitycertificate) | [ValidAcquireIssuanceCertificateArgs](#interface-validacquireissuancecertificateargs) | [WalletEncryptionArgs](#interface-walletencryptionargs) | +| [IdentityCertifier](#interface-identitycertifier) | [ValidBasketInsertion](#interface-validbasketinsertion) | [WalletErrorObject](#interface-walleterrorobject) | +| [InternalizeActionArgs](#interface-internalizeactionargs) | [ValidCreateActionArgs](#interface-validcreateactionargs) | [WalletInterface](#interface-walletinterface) | +| [InternalizeActionResult](#interface-internalizeactionresult) | [ValidCreateActionInput](#interface-validcreateactioninput) | [WalletLoggerInterface](#interface-walletloggerinterface) | +| [InternalizeOutput](#interface-internalizeoutput) | [ValidCreateActionOptions](#interface-validcreateactionoptions) | [WalletLoggerLog](#interface-walletloggerlog) | +| [KeyDeriverApi](#interface-keyderiverapi) | [ValidCreateActionOutput](#interface-validcreateactionoutput) | [WalletOutput](#interface-walletoutput) | +| [KeyLinkageResult](#interface-keylinkageresult) | [ValidDiscoverByAttributesArgs](#interface-validdiscoverbyattributesargs) | [WalletPayment](#interface-walletpayment) | +| [ListActionsArgs](#interface-listactionsargs) | [ValidDiscoverByIdentityKeyArgs](#interface-validdiscoverbyidentitykeyargs) | [WalletWire](#interface-walletwire) | +| [ListActionsResult](#interface-listactionsresult) | [ValidInternalizeActionArgs](#interface-validinternalizeactionargs) | | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -60,23 +60,9 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( --- ### Interface: AbortActionResult -Result of an `abortAction` call. - -`aborted` is informative: `true` indicates the wallet successfully invalidated -the action (it will not be broadcast and its inputs are released), `false` -indicates the wallet refused to abort because the underlying transaction was -found to already be on chain (mined or known to mempool). On a refusal the -caller should typically invoke `internalizeAction` instead, which will treat -the call as explicit authorization to advance the nosend lifecycle. - -Note that confirming on-chain status requires network reachability. When -confirmation is impossible (services unreachable or returning errors), the -wallet proceeds with the abort and returns `aborted: true` rather than -refusing — refusal is reserved for positive on-chain confirmation. - ```ts export interface AbortActionResult { - aborted: boolean; + aborted: true; } ``` @@ -237,27 +223,6 @@ export interface CreateActionResult { See also: [AtomicBEEF](./wallet.md#type-atomicbeef), [OutpointString](./wallet.md#type-outpointstring), [SendWithResult](./wallet.md#interface-sendwithresult), [SignableTransaction](./wallet.md#interface-signabletransaction), [TXIDHexString](./wallet.md#type-txidhexstring) -#### Property tx - -AtomicBEEF-encoded transaction bytes (BRC-95), produced by the wallet's -`createAction` / `signAction` methods (`wallet-toolbox` -`signer/methods/createAction.ts:66` and `signer/methods/signAction.ts:35`, -both invoking `beef.toBinaryAtomic(txid)`). The envelope carries the -broadcast transaction plus every input's `sourceTransaction` chain — a -subsequent broadcaster does not need to re-fetch parents to construct EF -or BEEF wire format. - -Parse with `Transaction.fromAtomicBEEF(tx)`. Treating these bytes as raw -tx binary (e.g. `Transaction.fromBinary(tx)`) will fail or produce a -transaction with empty `sourceTransaction` on every input. - -Absent when `options.returnTXIDOnly === true`. - -```ts -tx?: AtomicBEEF -``` -See also: [AtomicBEEF](./wallet.md#type-atomicbeef) - Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- @@ -539,16 +504,13 @@ export interface KeyDeriverApi { identityKey: string; derivePublicKey: (protocolID: WalletProtocol, keyID: string, counterparty: Counterparty, forSelf?: boolean) => PublicKey; derivePrivateKey: (protocolID: WalletProtocol, keyID: string, counterparty: Counterparty) => PrivateKey; - derivePrivateKeys?: (derivations: readonly PrivateKeyDerivation[]) => PrivateKey[]; deriveSymmetricKey: (protocolID: WalletProtocol, keyID: string, counterparty: Counterparty) => SymmetricKey; - derivePublicKeyAsync?: (protocolID: WalletProtocol, keyID: string, counterparty: Counterparty, forSelf?: boolean) => Promise; - deriveSymmetricKeyAsync?: (protocolID: WalletProtocol, keyID: string, counterparty: Counterparty) => Promise; revealCounterpartySecret: (counterparty: Counterparty) => number[]; revealSpecificSecret: (counterparty: Counterparty, protocolID: WalletProtocol, keyID: string) => number[]; } ``` -See also: [Counterparty](./wallet.md#type-counterparty), [PrivateKey](./primitives.md#class-privatekey), [PrivateKeyDerivation](./wallet.md#interface-privatekeyderivation), [PublicKey](./primitives.md#class-publickey), [SymmetricKey](./primitives.md#class-symmetrickey), [WalletProtocol](./wallet.md#type-walletprotocol) +See also: [Counterparty](./wallet.md#type-counterparty), [PrivateKey](./primitives.md#class-privatekey), [PublicKey](./primitives.md#class-publickey), [SymmetricKey](./primitives.md#class-symmetrickey), [WalletProtocol](./wallet.md#type-walletprotocol) #### Property derivePrivateKey @@ -559,17 +521,6 @@ derivePrivateKey: (protocolID: WalletProtocol, keyID: string, counterparty: Coun ``` See also: [Counterparty](./wallet.md#type-counterparty), [PrivateKey](./primitives.md#class-privatekey), [WalletProtocol](./wallet.md#type-walletprotocol) -#### Property derivePrivateKeys - -Derives several private keys while sharing each counterparty ECDH result. -Implementations that do not provide this additive lane retain the -per-key `derivePrivateKey` contract. - -```ts -derivePrivateKeys?: (derivations: readonly PrivateKeyDerivation[]) => PrivateKey[] -``` -See also: [PrivateKey](./primitives.md#class-privatekey), [PrivateKeyDerivation](./wallet.md#interface-privatekeyderivation) - #### Property derivePublicKey Derives a public key based on protocol ID, key ID, and counterparty. @@ -579,15 +530,6 @@ derivePublicKey: (protocolID: WalletProtocol, keyID: string, counterparty: Count ``` See also: [Counterparty](./wallet.md#type-counterparty), [PublicKey](./primitives.md#class-publickey), [WalletProtocol](./wallet.md#type-walletprotocol) -#### Property derivePublicKeyAsync - -Asynchronous acceleration lane; synchronous-only implementations may omit it. - -```ts -derivePublicKeyAsync?: (protocolID: WalletProtocol, keyID: string, counterparty: Counterparty, forSelf?: boolean) => Promise -``` -See also: [Counterparty](./wallet.md#type-counterparty), [PublicKey](./primitives.md#class-publickey), [WalletProtocol](./wallet.md#type-walletprotocol) - #### Property deriveSymmetricKey Derives a symmetric key based on protocol ID, key ID, and counterparty. @@ -598,15 +540,6 @@ deriveSymmetricKey: (protocolID: WalletProtocol, keyID: string, counterparty: Co ``` See also: [Counterparty](./wallet.md#type-counterparty), [SymmetricKey](./primitives.md#class-symmetrickey), [WalletProtocol](./wallet.md#type-walletprotocol) -#### Property deriveSymmetricKeyAsync - -Asynchronous acceleration lane; synchronous-only implementations may omit it. - -```ts -deriveSymmetricKeyAsync?: (protocolID: WalletProtocol, keyID: string, counterparty: Counterparty) => Promise -``` -See also: [Counterparty](./wallet.md#type-counterparty), [SymmetricKey](./primitives.md#class-symmetrickey), [WalletProtocol](./wallet.md#type-walletprotocol) - #### Property identityKey The identity of this key deriver which is normally the public key associated with the `rootKey` @@ -798,21 +731,6 @@ vout: number Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) ---- -### Interface: PrivateKeyDerivation - -```ts -export interface PrivateKeyDerivation { - protocolID: WalletProtocol; - keyID: string; - counterparty: Counterparty; -} -``` - -See also: [Counterparty](./wallet.md#type-counterparty), [WalletProtocol](./wallet.md#type-walletprotocol) - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - --- ### Interface: ProveCertificateArgs @@ -1077,21 +995,6 @@ export interface SignableTransaction { See also: [AtomicBEEF](./wallet.md#type-atomicbeef), [Base64String](./wallet.md#type-base64string) -#### Property tx - -AtomicBEEF-encoded transaction bytes (BRC-95). The envelope carries the -unsigned transaction plus every input's `sourceTransaction` chain so a -caller can complete signing without a separate parent-fetch round-trip. - -Parse with `Transaction.fromAtomicBEEF(tx)`. Treating these bytes as raw -tx binary (e.g. `Transaction.fromBinary(tx)`) will fail or produce a -transaction with empty `sourceTransaction` on every input. - -```ts -tx: AtomicBEEF -``` -See also: [AtomicBEEF](./wallet.md#type-atomicbeef) - Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- @@ -2289,15 +2192,9 @@ export default interface WalletWire { } ``` -#### Property transmitToWalletUint8Array - -Optional compact-byte transport. Implementations can provide this lane to -avoid boxing multi-megabyte wire frames while the legacy method remains -available for backwards compatibility. - -```ts -transmitToWalletUint8Array?: (message: Uint8Array) => Promise -``` +`transmitToWalletUint8Array` is an optional compact-byte lane for large frames. +The required `number[]` method and all wallet protocol contracts remain +unchanged. Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -2333,17 +2230,14 @@ export default class CachedKeyDeriver implements KeyDeriverApi { maxCacheSize?: number; }) derivePublicKey(protocolID: WalletProtocol, keyID: string, counterparty: Counterparty, forSelf: boolean = false): PublicKey - async derivePublicKeyAsync(protocolID: WalletProtocol, keyID: string, counterparty: Counterparty, forSelf: boolean = false): Promise derivePrivateKey(protocolID: WalletProtocol, keyID: string, counterparty: Counterparty): PrivateKey - derivePrivateKeys(derivations: readonly PrivateKeyDerivation[]): PrivateKey[] deriveSymmetricKey(protocolID: WalletProtocol, keyID: string, counterparty: Counterparty): SymmetricKey - async deriveSymmetricKeyAsync(protocolID: WalletProtocol, keyID: string, counterparty: Counterparty): Promise revealCounterpartySecret(counterparty: Counterparty): number[] revealSpecificSecret(counterparty: Counterparty, protocolID: WalletProtocol, keyID: string): number[] } ``` -See also: [Counterparty](./wallet.md#type-counterparty), [KeyDeriverApi](./wallet.md#interface-keyderiverapi), [PrivateKey](./primitives.md#class-privatekey), [PrivateKeyDerivation](./wallet.md#interface-privatekeyderivation), [PublicKey](./primitives.md#class-publickey), [SymmetricKey](./primitives.md#class-symmetrickey), [WalletProtocol](./wallet.md#type-walletprotocol) +See also: [Counterparty](./wallet.md#type-counterparty), [KeyDeriverApi](./wallet.md#interface-keyderiverapi), [PrivateKey](./primitives.md#class-privatekey), [PublicKey](./primitives.md#class-publickey), [SymmetricKey](./primitives.md#class-symmetrickey), [WalletProtocol](./wallet.md#type-walletprotocol) #### Constructor @@ -2541,7 +2435,7 @@ export default class HTTPWalletJSON implements WalletInterface { keyID?: KeyIDStringUnder800Bytes; privileged?: BooleanDefaultFalse; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; forSelf?: BooleanDefaultFalse; }): Promise<{ publicKey: PubKeyHex; @@ -2591,7 +2485,7 @@ export default class HTTPWalletJSON implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }): Promise<{ ciphertext: Byte[]; @@ -2605,7 +2499,7 @@ export default class HTTPWalletJSON implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }): Promise<{ plaintext: Byte[]; @@ -2619,7 +2513,7 @@ export default class HTTPWalletJSON implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }): Promise<{ hmac: Byte[]; @@ -2634,7 +2528,7 @@ export default class HTTPWalletJSON implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }): Promise<{ valid: true; @@ -2649,7 +2543,7 @@ export default class HTTPWalletJSON implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }): Promise<{ signature: Byte[]; @@ -2665,7 +2559,7 @@ export default class HTTPWalletJSON implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; forSelf?: BooleanDefaultFalse; privileged?: BooleanDefaultFalse; }): Promise<{ @@ -2735,9 +2629,9 @@ export default class HTTPWalletWire implements WalletWire { baseUrl: string; httpClient: typeof fetch; originator: string | undefined; - constructor(originator: string | undefined, baseUrl: string = "http://localhost:3301", httpClient?: typeof fetch) + constructor(originator: string | undefined, baseUrl: string = "http://localhost:3301", httpClient = fetch) async transmitToWallet(message: number[]): Promise - async transmitToWalletUint8Array(message: Uint8Array): Promise + async transmitToWalletUint8Array(message: Uint8Array): Promise } ``` @@ -2805,16 +2699,13 @@ export class KeyDeriver implements KeyDeriverApi { constructor(rootKey: PrivateKey | "anyone", private readonly cacheSharedSecret?: ((priv: PrivateKey, pub: Point, point: Point) => void), private readonly retrieveCachedSharedSecret?: ((priv: PrivateKey, pub: Point) => (Point | undefined))) derivePublicKey(protocolID: WalletProtocol, keyID: string, counterparty: Counterparty, forSelf: boolean = false): PublicKey derivePrivateKey(protocolID: WalletProtocol, keyID: string, counterparty: Counterparty): PrivateKey - derivePrivateKeys(derivations: readonly PrivateKeyDerivation[]): PrivateKey[] deriveSymmetricKey(protocolID: WalletProtocol, keyID: string, counterparty: Counterparty): SymmetricKey - async derivePublicKeyAsync(protocolID: WalletProtocol, keyID: string, counterparty: Counterparty, forSelf: boolean = false): Promise - async deriveSymmetricKeyAsync(protocolID: WalletProtocol, keyID: string, counterparty: Counterparty): Promise revealCounterpartySecret(counterparty: Counterparty): number[] revealSpecificSecret(counterparty: Counterparty, protocolID: WalletProtocol, keyID: string): number[] } ``` -See also: [Counterparty](./wallet.md#type-counterparty), [KeyDeriverApi](./wallet.md#interface-keyderiverapi), [Point](./primitives.md#class-point), [PrivateKey](./primitives.md#class-privatekey), [PrivateKeyDerivation](./wallet.md#interface-privatekeyderivation), [PublicKey](./primitives.md#class-publickey), [SymmetricKey](./primitives.md#class-symmetrickey), [WalletProtocol](./wallet.md#type-walletprotocol) +See also: [Counterparty](./wallet.md#type-counterparty), [KeyDeriverApi](./wallet.md#interface-keyderiverapi), [Point](./primitives.md#class-point), [PrivateKey](./primitives.md#class-privatekey), [PublicKey](./primitives.md#class-publickey), [SymmetricKey](./primitives.md#class-symmetrickey), [WalletProtocol](./wallet.md#type-walletprotocol) #### Constructor @@ -2965,7 +2856,7 @@ export class ProtoWallet { async revealCounterpartyKeyLinkage(args: RevealCounterpartyKeyLinkageArgs): Promise async revealSpecificKeyLinkage(args: RevealSpecificKeyLinkageArgs): Promise async encrypt(args: WalletEncryptArgs): Promise - async decrypt(args: WalletDecryptArgs, _originator?: string): Promise + async decrypt(args: WalletDecryptArgs, originator?: string): Promise async createHmac(args: CreateHmacArgs): Promise async verifyHmac(args: VerifyHmacArgs): Promise async createSignature(args: CreateSignatureArgs): Promise @@ -2982,22 +2873,9 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( Facilitates wallet operations over cross-document messaging. -A React Native host answers a BRC-100 invocation by injecting the response -into the document that made the call, so a response is delivered by this -window, by the frame bridging for it, or by a host-synthesized event that -carries no source at all. Messages from any other browsing context - a -framed document, an opener, or a sandboxed frame reporting an opaque origin -- are never wallet responses and are ignored before their payload is read. -A relaying host frame is a separate browsing context, so its browser-attested -origin must belong to this document or to the configured wallet origin. -Whatever origin a host stamps on an event it synthesizes in this document is -accepted, because the browser does not attest it and the injection is already -same-origin; configuring an exact domain additionally pins every response to -that origin, while the default wildcard target keeps every host reachable. - ```ts export default class ReactNativeWebView extends InvokableWalletBase { - constructor(domain: string = "*", responseTimeout?: number) + constructor(domain: string = "*") async invoke(call: CallType, args: any): Promise } ``` @@ -3100,7 +2978,7 @@ export default class WalletClient implements WalletInterface { async abortAction(args: { reference: Base64String; }): Promise<{ - aborted: boolean; + aborted: true; }> async listActions(args: ListActionsArgs): Promise async internalizeAction(args: InternalizeActionArgs): Promise<{ @@ -3122,7 +3000,7 @@ export default class WalletClient implements WalletInterface { keyID?: KeyIDStringUnder800Bytes; privileged?: BooleanDefaultFalse; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; forSelf?: BooleanDefaultFalse; }): Promise<{ publicKey: PubKeyHex; @@ -3171,7 +3049,7 @@ export default class WalletClient implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }): Promise<{ ciphertext: Byte[]; @@ -3184,7 +3062,7 @@ export default class WalletClient implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }): Promise<{ plaintext: Byte[]; @@ -3197,7 +3075,7 @@ export default class WalletClient implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }): Promise<{ hmac: Byte[]; @@ -3211,7 +3089,7 @@ export default class WalletClient implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }): Promise<{ valid: true; @@ -3225,7 +3103,7 @@ export default class WalletClient implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }): Promise<{ signature: Byte[]; @@ -3240,7 +3118,7 @@ export default class WalletClient implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; forSelf?: BooleanDefaultFalse; privileged?: BooleanDefaultFalse; }): Promise<{ @@ -3338,7 +3216,7 @@ export default class WalletWireProcessor implements WalletWire { wallet: WalletInterface; constructor(wallet: WalletInterface) async transmitToWallet(message: number[]): Promise - async transmitToWalletUint8Array(message: Uint8Array): Promise + async transmitToWalletUint8Array(message: Uint8Array): Promise } ``` @@ -3383,7 +3261,7 @@ export default class WalletWireTransceiver implements WalletInterface { keyID?: KeyIDStringUnder800Bytes; privileged?: BooleanDefaultFalse; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; forSelf?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ publicKey: PubKeyHex; @@ -3433,7 +3311,7 @@ export default class WalletWireTransceiver implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ ciphertext: Byte[]; @@ -3447,7 +3325,7 @@ export default class WalletWireTransceiver implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ plaintext: Byte[]; @@ -3461,7 +3339,7 @@ export default class WalletWireTransceiver implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ hmac: Byte[]; @@ -3476,7 +3354,7 @@ export default class WalletWireTransceiver implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ valid: true; @@ -3491,7 +3369,7 @@ export default class WalletWireTransceiver implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ signature: Byte[]; @@ -3507,7 +3385,7 @@ export default class WalletWireTransceiver implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; forSelf?: BooleanDefaultFalse; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ @@ -3621,11 +3499,32 @@ export default class WindowCWISubstrate implements WalletInterface { reference: Base64String; }; }> - async signAction(args: SignActionArgs, originator?: OriginatorDomainNameStringUnder250Bytes): Promise + async signAction(args: { + spends: Record; + reference: Base64String; + options?: { + acceptDelayedBroadcast?: BooleanDefaultTrue; + returnTXIDOnly?: BooleanDefaultFalse; + noSend?: BooleanDefaultFalse; + noSendChange?: OutpointString[]; + sendWith: TXIDHexString[]; + }; + }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ + txid?: TXIDHexString; + tx?: BEEF; + noSendChange?: OutpointString[]; + sendWithResults?: Array<{ + txid: TXIDHexString; + status: "unproven" | "sending" | "failed"; + }>; + }> async abortAction(args: { reference: Base64String; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ - aborted: boolean; + aborted: true; }> async listActions(args: { labels: LabelStringUnder300Bytes[]; @@ -3728,7 +3627,7 @@ export default class WindowCWISubstrate implements WalletInterface { keyID?: KeyIDStringUnder800Bytes; privileged?: BooleanDefaultFalse; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; forSelf?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ publicKey: PubKeyHex; @@ -3777,7 +3676,7 @@ export default class WindowCWISubstrate implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ ciphertext: Byte[]; @@ -3790,7 +3689,7 @@ export default class WindowCWISubstrate implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ plaintext: Byte[]; @@ -3803,7 +3702,7 @@ export default class WindowCWISubstrate implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ hmac: Byte[]; @@ -3817,7 +3716,7 @@ export default class WindowCWISubstrate implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ valid: true; @@ -3831,7 +3730,7 @@ export default class WindowCWISubstrate implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ signature: Byte[]; @@ -3846,13 +3745,33 @@ export default class WindowCWISubstrate implements WalletInterface { ]; keyID: KeyIDStringUnder800Bytes; privilegedReason?: DescriptionString5to50Bytes; - counterparty?: PubKeyHex; + counterparty?: PubKeyHex | "self" | "anyone"; forSelf?: BooleanDefaultFalse; privileged?: BooleanDefaultFalse; }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ valid: true; }> - async acquireCertificate(args: AcquireCertificateArgs, originator?: OriginatorDomainNameStringUnder250Bytes): Promise + async acquireCertificate(args: { + type: Base64String; + subject: PubKeyHex; + serialNumber: Base64String; + revocationOutpoint: OutpointString; + signature: HexString; + fields: Record; + certifier: PubKeyHex; + keyringRevealer: PubKeyHex | "certifier"; + keyringForSubject: Record; + acquisitionProtocol: "direct" | "issuance"; + certifierUrl?: string; + }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ + type: Base64String; + subject: PubKeyHex; + serialNumber: Base64String; + certifier: PubKeyHex; + revocationOutpoint: OutpointString; + signature: HexString; + fields: Record; + }> async listCertificates(args: { certifiers: PubKeyHex[]; types: Base64String[]; @@ -3872,7 +3791,23 @@ export default class WindowCWISubstrate implements WalletInterface { fields: Record; }>; }> - async proveCertificate(args: ProveCertificateArgs, originator?: OriginatorDomainNameStringUnder250Bytes): Promise + async proveCertificate(args: { + certificate: { + type: Base64String; + subject: PubKeyHex; + serialNumber: Base64String; + certifier: PubKeyHex; + revocationOutpoint: OutpointString; + signature: HexString; + fields: Record; + }; + fieldsToReveal: CertificateFieldNameUnder50Bytes[]; + verifier: PubKeyHex; + privileged?: BooleanDefaultFalse; + privilegedReason?: DescriptionString5to50Bytes; + }, originator?: OriginatorDomainNameStringUnder250Bytes): Promise<{ + keyringForVerifier: Record; + }> async relinquishCertificate(args: { type: Base64String; serialNumber: Base64String; @@ -3951,7 +3886,7 @@ export default class WindowCWISubstrate implements WalletInterface { } ``` -See also: [AcquireCertificateArgs](./wallet.md#interface-acquirecertificateargs), [BEEF](./wallet.md#type-beef), [Base64String](./wallet.md#type-base64string), [BasketStringUnder300Bytes](./wallet.md#type-basketstringunder300bytes), [BooleanDefaultFalse](./wallet.md#type-booleandefaultfalse), [BooleanDefaultTrue](./wallet.md#type-booleandefaulttrue), [Byte](./wallet.md#type-byte), [CertificateFieldNameUnder50Bytes](./wallet.md#type-certificatefieldnameunder50bytes), [DescriptionString5to50Bytes](./wallet.md#type-descriptionstring5to50bytes), [EntityIconURLStringMax500Bytes](./wallet.md#type-entityiconurlstringmax500bytes), [EntityNameStringMax100Bytes](./wallet.md#type-entitynamestringmax100bytes), [HexString](./wallet.md#type-hexstring), [ISOTimestampString](./wallet.md#type-isotimestampstring), [KeyIDStringUnder800Bytes](./wallet.md#type-keyidstringunder800bytes), [LabelStringUnder300Bytes](./wallet.md#type-labelstringunder300bytes), [OriginatorDomainNameStringUnder250Bytes](./wallet.md#type-originatordomainnamestringunder250bytes), [OutpointString](./wallet.md#type-outpointstring), [OutputTagStringUnder300Bytes](./wallet.md#type-outputtagstringunder300bytes), [PositiveInteger](./wallet.md#type-positiveinteger), [PositiveIntegerDefault10Max10000](./wallet.md#type-positiveintegerdefault10max10000), [PositiveIntegerMax10](./wallet.md#type-positiveintegermax10), [PositiveIntegerOrZero](./wallet.md#type-positiveintegerorzero), [ProtocolString5To400Bytes](./wallet.md#type-protocolstring5to400bytes), [ProveCertificateArgs](./wallet.md#interface-provecertificateargs), [ProveCertificateResult](./wallet.md#interface-provecertificateresult), [PubKeyHex](./wallet.md#type-pubkeyhex), [SatoshiValue](./wallet.md#type-satoshivalue), [SecurityLevel](./wallet.md#type-securitylevel), [SignActionArgs](./wallet.md#interface-signactionargs), [SignActionResult](./wallet.md#interface-signactionresult), [TXIDHexString](./wallet.md#type-txidhexstring), [VersionString7To30Bytes](./wallet.md#type-versionstring7to30bytes), [WalletCertificate](./wallet.md#interface-walletcertificate), [WalletInterface](./wallet.md#interface-walletinterface), [decrypt](./messages.md#variable-decrypt), [encrypt](./messages.md#variable-encrypt) +See also: [BEEF](./wallet.md#type-beef), [Base64String](./wallet.md#type-base64string), [BasketStringUnder300Bytes](./wallet.md#type-basketstringunder300bytes), [BooleanDefaultFalse](./wallet.md#type-booleandefaultfalse), [BooleanDefaultTrue](./wallet.md#type-booleandefaulttrue), [Byte](./wallet.md#type-byte), [CertificateFieldNameUnder50Bytes](./wallet.md#type-certificatefieldnameunder50bytes), [DescriptionString5to50Bytes](./wallet.md#type-descriptionstring5to50bytes), [EntityIconURLStringMax500Bytes](./wallet.md#type-entityiconurlstringmax500bytes), [EntityNameStringMax100Bytes](./wallet.md#type-entitynamestringmax100bytes), [HexString](./wallet.md#type-hexstring), [ISOTimestampString](./wallet.md#type-isotimestampstring), [KeyIDStringUnder800Bytes](./wallet.md#type-keyidstringunder800bytes), [LabelStringUnder300Bytes](./wallet.md#type-labelstringunder300bytes), [OriginatorDomainNameStringUnder250Bytes](./wallet.md#type-originatordomainnamestringunder250bytes), [OutpointString](./wallet.md#type-outpointstring), [OutputTagStringUnder300Bytes](./wallet.md#type-outputtagstringunder300bytes), [PositiveInteger](./wallet.md#type-positiveinteger), [PositiveIntegerDefault10Max10000](./wallet.md#type-positiveintegerdefault10max10000), [PositiveIntegerMax10](./wallet.md#type-positiveintegermax10), [PositiveIntegerOrZero](./wallet.md#type-positiveintegerorzero), [ProtocolString5To400Bytes](./wallet.md#type-protocolstring5to400bytes), [PubKeyHex](./wallet.md#type-pubkeyhex), [SatoshiValue](./wallet.md#type-satoshivalue), [SecurityLevel](./wallet.md#type-securitylevel), [TXIDHexString](./wallet.md#type-txidhexstring), [VersionString7To30Bytes](./wallet.md#type-versionstring7to30bytes), [WalletInterface](./wallet.md#interface-walletinterface), [decrypt](./messages.md#variable-decrypt), [encrypt](./messages.md#variable-encrypt) Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) @@ -3960,11 +3895,6 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( Facilitates wallet operations over cross-document messaging. -The default wildcard target supports wallets embedded by public web apps, -including callers with opaque origins. Configure an exact origin when the -parent is known. Responses must always come from the current parent window; -exact-origin mode additionally requires the configured origin. - ```ts export default class XDMSubstrate extends InvokableWalletBase { constructor(domain: string = "*") @@ -3981,35 +3911,22 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( | | | | | --- | --- | --- | -| [brc100JsonReplacer](#function-brc100jsonreplacer) | [validateBasketInsertion](#function-validatebasketinsertion) | [validateOptionalInteger](#function-validateoptionalinteger) | -| [isHexString](#function-ishexstring) | [validateCreateActionArgs](#function-validatecreateactionargs) | [validateOptionalOutpointString](#function-validateoptionaloutpointstring) | -| [normalizeBRC100ByteArray](#function-normalizebrc100bytearray) | [validateCreateActionInput](#function-validatecreateactioninput) | [validateOriginator](#function-validateoriginator) | -| [normalizeBRC100ByteFields](#function-normalizebrc100bytefields) | [validateCreateActionOptions](#function-validatecreateactionoptions) | [validateOutpointString](#function-validateoutpointstring) | -| [normalizeBRC100WalletByteFields](#function-normalizebrc100walletbytefields) | [validateCreateActionOutput](#function-validatecreateactionoutput) | [validatePositiveIntegerOrZero](#function-validatepositiveintegerorzero) | -| [parseWalletOutpoint](#function-parsewalletoutpoint) | [validateDiscoverByAttributesArgs](#function-validatediscoverbyattributesargs) | [validateProveCertificateArgs](#function-validateprovecertificateargs) | -| [stringifyBRC100](#function-stringifybrc100) | [validateDiscoverByIdentityKeyArgs](#function-validatediscoverbyidentitykeyargs) | [validateRelinquishCertificateArgs](#function-validaterelinquishcertificateargs) | -| [toBRC100PortableByteArray](#function-tobrc100portablebytearray) | [validateInteger](#function-validateinteger) | [validateRelinquishOutputArgs](#function-validaterelinquishoutputargs) | -| [toOriginHeader](#function-tooriginheader) | [validateInternalizeActionArgs](#function-validateinternalizeactionargs) | [validateSatoshis](#function-validatesatoshis) | -| [validateAbortActionArgs](#function-validateabortactionargs) | [validateInternalizeOutput](#function-validateinternalizeoutput) | [validateSignActionArgs](#function-validatesignactionargs) | -| [validateAcquireDirectCertificateArgs](#function-validateacquiredirectcertificateargs) | [validateListActionsArgs](#function-validatelistactionsargs) | [validateSignActionOptions](#function-validatesignactionoptions) | -| [validateAcquireIssuanceCertificateArgs](#function-validateacquireissuancecertificateargs) | [validateListCertificatesArgs](#function-validatelistcertificatesargs) | [validateStringLength](#function-validatestringlength) | -| [validateBase64String](#function-validatebase64string) | [validateListOutputsArgs](#function-validatelistoutputsargs) | [validateWalletPayment](#function-validatewalletpayment) | +| [isHexString](#function-ishexstring) | [validateCreateActionOutput](#function-validatecreateactionoutput) | [validateOriginator](#function-validateoriginator) | +| [parseWalletOutpoint](#function-parsewalletoutpoint) | [validateDiscoverByAttributesArgs](#function-validatediscoverbyattributesargs) | [validateOutpointString](#function-validateoutpointstring) | +| [toOriginHeader](#function-tooriginheader) | [validateDiscoverByIdentityKeyArgs](#function-validatediscoverbyidentitykeyargs) | [validatePositiveIntegerOrZero](#function-validatepositiveintegerorzero) | +| [validateAbortActionArgs](#function-validateabortactionargs) | [validateInteger](#function-validateinteger) | [validateProveCertificateArgs](#function-validateprovecertificateargs) | +| [validateAcquireDirectCertificateArgs](#function-validateacquiredirectcertificateargs) | [validateInternalizeActionArgs](#function-validateinternalizeactionargs) | [validateRelinquishCertificateArgs](#function-validaterelinquishcertificateargs) | +| [validateAcquireIssuanceCertificateArgs](#function-validateacquireissuancecertificateargs) | [validateInternalizeOutput](#function-validateinternalizeoutput) | [validateRelinquishOutputArgs](#function-validaterelinquishoutputargs) | +| [validateBase64String](#function-validatebase64string) | [validateListActionsArgs](#function-validatelistactionsargs) | [validateSatoshis](#function-validatesatoshis) | +| [validateBasketInsertion](#function-validatebasketinsertion) | [validateListCertificatesArgs](#function-validatelistcertificatesargs) | [validateSignActionArgs](#function-validatesignactionargs) | +| [validateCreateActionArgs](#function-validatecreateactionargs) | [validateListOutputsArgs](#function-validatelistoutputsargs) | [validateSignActionOptions](#function-validatesignactionoptions) | +| [validateCreateActionInput](#function-validatecreateactioninput) | [validateOptionalInteger](#function-validateoptionalinteger) | [validateStringLength](#function-validatestringlength) | +| [validateCreateActionOptions](#function-validatecreateactionoptions) | [validateOptionalOutpointString](#function-validateoptionaloutpointstring) | [validateWalletPayment](#function-validatewalletpayment) | Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) --- -### Function: brc100JsonReplacer - -JSON replacer that preserves `Uint8Array` values as portable JSON arrays. - -```ts -export function brc100JsonReplacer(this: Record, key: string, value: unknown): unknown -``` - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- ### Function: isHexString Check whether a string is a valid hex string (even length and hex characters). @@ -4029,50 +3946,6 @@ Argument Details Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) ---- -### Function: normalizeBRC100ByteArray - -Normalizes the runtime representations used for BRC-100 byte arrays. - -Healthy `number[]` and `Uint8Array` values are returned by identity so the -common path does not allocate. The fallback recovers the contiguous -numeric-key object produced by `JSON.stringify(new Uint8Array(...))` in -historical JSON transports. Invalid, sparse, or non-byte input is rejected. - -```ts -export function normalizeBRC100ByteArray(value: unknown): AtomicBEEF | undefined -``` - -See also: [AtomicBEEF](./wallet.md#type-atomicbeef) - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- -### Function: normalizeBRC100ByteFields - -Repairs byte arrays only in explicitly selected own fields of one protocol -object. This non-recursive helper is for envelopes that also contain opaque -application data, where field-name-based traversal would be destructive. - -```ts -export function normalizeBRC100ByteFields(value: T, fieldNames: readonly string[]): T -``` - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- -### Function: normalizeBRC100WalletByteFields - -Repairs known byte fields in a wallet request, result, or serialized wallet -error. This is intentionally field-aware: unrelated numeric-key objects are -left untouched. Parsed JSON objects are normalized in place. - -```ts -export function normalizeBRC100WalletByteFields(value: T): T -``` - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - --- ### Function: parseWalletOutpoint @@ -4085,28 +3958,6 @@ export function parseWalletOutpoint(outpoint: string): { Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) ---- -### Function: stringifyBRC100 - -Serialize a BRC-100 payload without allowing typed byte arrays to become objects. - -```ts -export function stringifyBRC100(value: unknown, space?: string | number): string -``` - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - ---- -### Function: toBRC100PortableByteArray - -Convert a valid BRC-100 byte array to the portable JSON `number[]` form. - -```ts -export function toBRC100PortableByteArray(value: unknown): number[] | undefined -``` - -Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables) - --- ### Function: toOriginHeader @@ -4965,12 +4816,6 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( --- ### Type: BooleanDefaultFalse -Analyzer note: the named primitive aliases in this file are the public -BRC-100 wire vocabulary. Their names and JSDoc constraints feed generated -API documentation and communicate semantics that primitive types cannot. -S6564 is therefore suppressed on those declarations instead of erasing the -public protocol vocabulary. - ```ts export type BooleanDefaultFalse = boolean ``` @@ -5017,7 +4862,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Type: Counterparty ```ts -export type Counterparty = PublicKey | PubKeyHex +export type Counterparty = PublicKey | PubKeyHex | "self" | "anyone" ``` See also: [PubKeyHex](./wallet.md#type-pubkeyhex), [PublicKey](./primitives.md#class-publickey) @@ -5111,7 +4956,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Type: KeyringRevealer ```ts -export type KeyringRevealer = PubKeyHex +export type KeyringRevealer = PubKeyHex | "certifier" ``` See also: [PubKeyHex](./wallet.md#type-pubkeyhex) @@ -5310,7 +5155,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions]( ### Type: WalletCounterparty ```ts -export type WalletCounterparty = PubKeyHex +export type WalletCounterparty = PubKeyHex | "self" | "anyone" ``` See also: [PubKeyHex](./wallet.md#type-pubkeyhex) From 965d6759c3710b074a43d36f9051d604ca1ddcb4 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 15:14:34 -0500 Subject: [PATCH 128/168] docs(sdk): record the Merkle path offset correction in the release notes The ordered-merge integration took the incoming @bsv/sdk release-note entry verbatim and dropped this change's sentences. Append them to the cumulative candidate text and regenerate the package migration table. Co-Authored-By: Claude Fable 5.1 --- docs/reference/package-api-migrations.md | 6 +++--- governance/package-release-notes.json | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 4bd878df7..35cb5feaf 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -48,7 +48,7 @@ and clean-consumer tests remain the executable type authority. | `@bsv/overlay-topics` | `1.6.10` | `1.8.2` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update. | | `@bsv/paymail` | `2.4.2` | `2.4.8` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. Consumers of the former bundled Money Button or Tokenized specification documents must follow the authoritative links in docs/specs/README.md. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/payment-express-middleware` | `2.1.1` | `2.1.6` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | -| `@bsv/sdk` | `2.7.0` | `2.7.1` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No API or wire migration is required. Upgrade to @bsv/sdk 2.7.1 or later. Valid BRC-103 v0.1 peers keep the same message fields, payload bytes, nonces, and signature derivation; authenticated messages whose transport identity differs from the established session now fail closed. | +| `@bsv/sdk` | `2.7.0` | `2.7.1` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No API or wire migration is required. Upgrade to @bsv/sdk 2.7.1 or later. Valid BRC-103 v0.1 peers keep the same message fields, payload bytes, nonces, and signature derivation; authenticated messages whose transport identity differs from the established session now fail closed. No API or valid BUMP wire migration is required for the offset correction. | | `@bsv/simple` | `0.4.1` | `0.5.3` | minor | [API and usage](../packages/helpers/simple.md) | Existing overlay configurations and number-array behavior are unchanged. TTN consumers select network teratestnet; all consumers should upgrade to @bsv/sdk 2.4.2 or later. The Message Box Client dependency advance requires no helper API migration and preserves the default HTTP and live-socket behavior. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/. | | `@bsv/templates` | `1.9.1` | `1.10.1` | minor | [API and usage](../packages/helpers/templates.md) | No existing consumer migration is required; existing template APIs and generated scripts are unchanged. New R1K1Wallet consumers await lock(), retain each private 32-byte salt, and provide a PIV signer that signs the supplied digest directly without hashing it again. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/teranode-listener` | `1.1.1` | `1.1.5` | patch | [API and usage](../packages/network/teranode-listener.md) | No consumer migration is required; listener APIs, topics, and network configuration are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | @@ -359,8 +359,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/sdk/bsv-sdk.md](../packages/sdk/bsv-sdk.md) - Source: [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) -- Release note: Binds authenticated general messages, certificate requests, and certificate responses to the identity stored in the nonce-selected session. Mismatched transport identity metadata is rejected, and application callbacks receive only the identity used for signature verification. -- Migration: No API or wire migration is required. Upgrade to @bsv/sdk 2.7.1 or later. Valid BRC-103 v0.1 peers keep the same message fields, payload bytes, nonces, and signature derivation; authenticated messages whose transport identity differs from the established session now fail closed. +- Release note: Binds authenticated general messages, certificate requests, and certificate responses to the identity stored in the nonce-selected session. Mismatched transport identity metadata is rejected, and application callbacks receive only the identity used for signature verification. The current unpublished candidate also corrects Merkle-path arithmetic for the full safe-integer offset domain, including positions above 32 bits, and rejects nonrepresentable offsets. +- Migration: No API or wire migration is required. Upgrade to @bsv/sdk 2.7.1 or later. Valid BRC-103 v0.1 peers keep the same message fields, payload bytes, nonces, and signature derivation; authenticated messages whose transport identity differs from the established session now fail closed. No API or valid BUMP wire migration is required for the offset correction. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index 069a72bc1..91c7987e6 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -168,8 +168,8 @@ "name": "@bsv/sdk", "publishedVersion": "2.7.0", "releaseType": "patch", - "summary": "Binds authenticated general messages, certificate requests, and certificate responses to the identity stored in the nonce-selected session. Mismatched transport identity metadata is rejected, and application callbacks receive only the identity used for signature verification.", - "migration": "No API or wire migration is required. Upgrade to @bsv/sdk 2.7.1 or later. Valid BRC-103 v0.1 peers keep the same message fields, payload bytes, nonces, and signature derivation; authenticated messages whose transport identity differs from the established session now fail closed." + "summary": "Binds authenticated general messages, certificate requests, and certificate responses to the identity stored in the nonce-selected session. Mismatched transport identity metadata is rejected, and application callbacks receive only the identity used for signature verification. The current unpublished candidate also corrects Merkle-path arithmetic for the full safe-integer offset domain, including positions above 32 bits, and rejects nonrepresentable offsets.", + "migration": "No API or wire migration is required. Upgrade to @bsv/sdk 2.7.1 or later. Valid BRC-103 v0.1 peers keep the same message fields, payload bytes, nonces, and signature derivation; authenticated messages whose transport identity differs from the established session now fail closed. No API or valid BUMP wire migration is required for the offset correction." }, { "name": "@bsv/simple", From 82d9f4bb81618ba47542f3c0064b3971f364fe2b Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 15:36:19 -0500 Subject: [PATCH 129/168] fix(sdk): reject cancelled lookups instead of answering empty query() and queryDetailed() flatten query$ to its first snapshot. The terminal gate only threw when hostCount was 0, and explicitly skipped the throw for terminalReason 'cancelled', so a caller abort resolved as { type: 'output-list', outputs: [] }. At any host count an aborted attempt was indistinguishable from an authoritative "this service has no outputs" answer, which is exactly the completion-vs-empty confusion queryDetailed exists to prevent. Throw lookupAbortError() before the host-count gate whenever the terminal snapshot reports 'cancelled'. query$ is unchanged and still emits its terminalReason: 'cancelled' snapshot for progressive callers. Co-Authored-By: Claude Fable 5.1 --- packages/sdk/CHANGELOG.md | 4 ++ packages/sdk/docs/reference/overlay-tools.md | 11 ++++ .../sdk/src/overlay-tools/LookupResolver.ts | 25 ++++++-- .../__tests/LookupResolver.dynamic.test.ts | 57 +++++++++++++++++++ 4 files changed, 93 insertions(+), 4 deletions(-) diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 6f84b4ca6..f144a6a27 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -282,6 +282,10 @@ All notable changes to this project will be documented in this file. The format - LookupResolver host cache no longer lets a tighter-limit discovery satisfy a later larger query, and `query()` still throws the historical no-competent-hosts error when a deadline expires before any host is admitted. +- `LookupResolver.query()` and `queryDetailed()` now reject with an `AbortError` + when the caller's `options.signal` cancels the attempt, at any host count, + instead of flattening a cancelled run into an empty output list. `query$()` + still reports the cancellation as a `terminalReason: 'cancelled'` snapshot. - Batch BEEF mutation bookkeeping and reuse compound Merkle intermediate hashes. The optional asynchronous P2PKH backend now forwards its already validated compressed public key directly into the unlocking script. Existing BEEF diff --git a/packages/sdk/docs/reference/overlay-tools.md b/packages/sdk/docs/reference/overlay-tools.md index 4bd68bade..ff68d88dc 100644 --- a/packages/sdk/docs/reference/overlay-tools.md +++ b/packages/sdk/docs/reference/overlay-tools.md @@ -371,6 +371,10 @@ See also: [UnreachableHostInfo](./overlay-tools.md#interface-unreachablehostinfo #### Property signal Abort this query without cancelling discovery still owned by another query. +`query()` and `queryDetailed()` reject with an `AbortError` once this +signal fires: a cancelled attempt never answered the question, so it is +never reported as an empty output list. `query$()` keeps emitting its +terminal snapshot with `terminalReason: 'cancelled'` instead. ```ts signal?: AbortSignal @@ -965,6 +969,9 @@ Optional `options.graceMs` overrides the per-call grace window (default 80 ms). Optional `options.softTimeoutMs` resolves the query early with whatever has arrived once any host has answered (or with an empty result if no host has answered by `softTimeoutMs`). +Throws an `AbortError` when `options.signal` aborted the attempt, so a +cancelled lookup is never mistaken for an authoritative empty answer. + ```ts async query(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise ``` @@ -987,6 +994,10 @@ Performs a lookup and returns both its answer and the host settlement evidence required by security-sensitive consumers to distinguish an authoritative empty result from an availability failure. +Throws an `AbortError` when `options.signal` aborted the attempt, rather +than returning a resolution whose empty answer would have to be +re-qualified against `progress.terminalReason`. + ```ts async queryDetailed(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise ``` diff --git a/packages/sdk/src/overlay-tools/LookupResolver.ts b/packages/sdk/src/overlay-tools/LookupResolver.ts index 274d5f392..3e3c8b63d 100644 --- a/packages/sdk/src/overlay-tools/LookupResolver.ts +++ b/packages/sdk/src/overlay-tools/LookupResolver.ts @@ -60,7 +60,13 @@ export type LookupFacilitatorAnswer = LookupAnswer | LookupFreeformAnswer * All optional; defaults preserve prior behavior. */ export interface LookupQueryOptions { - /** Abort this query without cancelling discovery still owned by another query. */ + /** + * Abort this query without cancelling discovery still owned by another query. + * `query()` and `queryDetailed()` reject with an `AbortError` once this + * signal fires: a cancelled attempt never answered the question, so it is + * never reported as an empty output list. `query$()` keeps emitting its + * terminal snapshot with `terminalReason: 'cancelled'` instead. + */ signal?: AbortSignal /** * Callback intake budget, independent of legacy aggregation. Defaults to 512 @@ -968,6 +974,9 @@ export default class LookupResolver { * Optional `options.graceMs` overrides the per-call grace window (default 80 ms). * Optional `options.softTimeoutMs` resolves the query early with whatever has arrived once any host has * answered (or with an empty result if no host has answered by `softTimeoutMs`). + * + * Throws an `AbortError` when `options.signal` aborted the attempt, so a + * cancelled lookup is never mistaken for an authoritative empty answer. */ async query( question: LookupQuestion, @@ -981,6 +990,10 @@ export default class LookupResolver { * Performs a lookup and returns both its answer and the host settlement * evidence required by security-sensitive consumers to distinguish an * authoritative empty result from an availability failure. + * + * Throws an `AbortError` when `options.signal` aborted the attempt, rather + * than returning a resolution whose empty answer would have to be + * re-qualified against `progress.terminalReason`. */ async queryDetailed( question: LookupQuestion, @@ -1020,9 +1033,13 @@ export default class LookupResolver { terminalReason: 'settled', ...(options?.correlationId !== undefined ? { correlationId: options.correlationId } : {}) } - // Promise callers cannot see terminalReason. A deadline that admitted no - // host is a miss, not a successful empty answer from a queried host. - if (progress.hostCount === 0 && progress.terminalReason !== 'cancelled') { + // Promise callers cannot see terminalReason. A cancelled attempt never + // answered the question, so it must not flatten into an empty output list + // at any host count. + if (progress.terminalReason === 'cancelled') throw lookupAbortError() + // A deadline that admitted no host is a miss, not a successful empty + // answer from a queried host. + if (progress.hostCount === 0) { throw new Error( `No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${question.service}` ) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts index c6d78fa33..25e8c3cb5 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts @@ -238,6 +238,63 @@ describe('LookupResolver dynamic discovery', () => { await expect(iterator.next()).resolves.toEqual({ done: true, value: undefined }) }) + it('rejects query() with an AbortError instead of an empty answer when a queried host is aborted', async () => { + const host = 'https://abort-query.example' + const controller = new AbortController() + const lookup = jest.fn( + async (_url: string, _question: unknown, _timeout: unknown, signal?: AbortSignal) => + await new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_abort_query: [host] } + }) + const pending = resolver.query({ service: 'ls_abort_query', query: {} }, undefined, { + signal: controller.signal + }) + pending.catch(() => { + /* asserted below */ + }) + + await jest.advanceTimersByTimeAsync(1) + expect(lookup).toHaveBeenCalledTimes(1) + controller.abort(new Error('caller stopped lookup')) + await jest.advanceTimersByTimeAsync(1) + + await expect(pending).rejects.toMatchObject({ + name: 'AbortError', + message: 'Lookup cancelled' + }) + }) + + it('rejects queryDetailed() with an AbortError when the caller aborts before a host is admitted', async () => { + const controller = new AbortController() + controller.abort(new Error('caller stopped lookup')) + const lookup = jest.fn() + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + hostOverrides: { ls_abort_early: ['https://abort-early.example'] } + }) + const pending = resolver.queryDetailed( + { service: 'ls_abort_early', query: {} }, + undefined, + { signal: controller.signal } + ) + pending.catch(() => { + /* asserted below */ + }) + + await jest.advanceTimersByTimeAsync(1) + + await expect(pending).rejects.toMatchObject({ + name: 'AbortError', + message: 'Lookup cancelled' + }) + expect(lookup).not.toHaveBeenCalled() + }) + it('emits a deadline terminal snapshot when no host receipt arrives', async () => { const host = 'https://deadline.example' const lookup = jest.fn( From 7f8bb2c3b1cae7c2099b28cdd62a648c572fe838 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 15:39:26 -0500 Subject: [PATCH 130/168] fix(sdk): keep discovery resource limits out of the no-hosts error chargeLookupDiscoveryBytes can exhaust maxTotalBytes while SLAP trackers are still being read, before any host is admitted. That leaves the terminal snapshot with hostCount 0, terminalReason 'resource-limit' and the limit recorded in limitsHit. queryDetailed()'s hostCount gate did not look at terminalReason, so it threw the historical "No competent hosts found by the SLAP trackers" Error and discarded both fields. Security-sensitive Promise callers could not tell "the trackers named no hosts" from "this attempt ran out of its own byte budget". Branch the gate on terminalReason: 'resource-limit' throws LookupResourceLimitError carrying the first limit that was hit, while 'deadline' and a settled empty discovery keep the historical error and message unchanged. Co-Authored-By: Claude Fable 5.1 --- packages/sdk/CHANGELOG.md | 4 ++ packages/sdk/docs/reference/overlay-tools.md | 6 ++- .../sdk/src/overlay-tools/LookupResolver.ts | 14 ++++++- .../__tests/LookupResolver.dynamic.test.ts | 39 ++++++++++++++++++- 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index f144a6a27..fb724536d 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -286,6 +286,10 @@ All notable changes to this project will be documented in this file. The format when the caller's `options.signal` cancels the attempt, at any host count, instead of flattening a cancelled run into an empty output list. `query$()` still reports the cancellation as a `terminalReason: 'cancelled'` snapshot. +- A lookup that exhausts a client resource budget during SLAP discovery, before + any host is admitted, now throws `LookupResourceLimitError` naming the limit + instead of the historical no-competent-hosts error. That message is reserved + for a deadline or a settled attempt that genuinely found no host. - Batch BEEF mutation bookkeeping and reuse compound Merkle intermediate hashes. The optional asynchronous P2PKH backend now forwards its already validated compressed public key directly into the unlocking script. Existing BEEF diff --git a/packages/sdk/docs/reference/overlay-tools.md b/packages/sdk/docs/reference/overlay-tools.md index ff68d88dc..c1118dddb 100644 --- a/packages/sdk/docs/reference/overlay-tools.md +++ b/packages/sdk/docs/reference/overlay-tools.md @@ -996,7 +996,11 @@ authoritative empty result from an availability failure. Throws an `AbortError` when `options.signal` aborted the attempt, rather than returning a resolution whose empty answer would have to be -re-qualified against `progress.terminalReason`. +re-qualified against `progress.terminalReason`. When a client resource +budget was exhausted during SLAP discovery, before any host could be +admitted, it throws `LookupResourceLimitError` naming that limit; the +historical no-competent-hosts error is reserved for a deadline or a +settled attempt that genuinely found no host. ```ts async queryDetailed(question: LookupQuestion, timeout?: number, options?: LookupQueryOptions): Promise diff --git a/packages/sdk/src/overlay-tools/LookupResolver.ts b/packages/sdk/src/overlay-tools/LookupResolver.ts index 3e3c8b63d..a5ad0bb30 100644 --- a/packages/sdk/src/overlay-tools/LookupResolver.ts +++ b/packages/sdk/src/overlay-tools/LookupResolver.ts @@ -993,7 +993,11 @@ export default class LookupResolver { * * Throws an `AbortError` when `options.signal` aborted the attempt, rather * than returning a resolution whose empty answer would have to be - * re-qualified against `progress.terminalReason`. + * re-qualified against `progress.terminalReason`. When a client resource + * budget was exhausted during SLAP discovery, before any host could be + * admitted, it throws `LookupResourceLimitError` naming that limit; the + * historical no-competent-hosts error is reserved for a deadline or a + * settled attempt that genuinely found no host. */ async queryDetailed( question: LookupQuestion, @@ -1038,8 +1042,14 @@ export default class LookupResolver { // at any host count. if (progress.terminalReason === 'cancelled') throw lookupAbortError() // A deadline that admitted no host is a miss, not a successful empty - // answer from a queried host. + // answer from a queried host. An attempt that exhausted a client resource + // budget during discovery is a third outcome: the trackers were never + // given the chance to name a host, so it keeps its own error and limit + // rather than borrowing the no-competent-hosts message. if (progress.hostCount === 0) { + if (progress.terminalReason === 'resource-limit') { + throw new LookupResourceLimitError(progress.limitsHit?.[0] ?? 'resource-limit') + } throw new Error( `No competent ${this.networkPreset} hosts found by the SLAP trackers for lookup service: ${question.service}` ) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts index 25e8c3cb5..f90b61755 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.dynamic.test.ts @@ -1,6 +1,7 @@ import LookupResolver, { HTTPSOverlayLookupFacilitator, - LookupAnswerProgress + LookupAnswerProgress, + LookupResourceLimitError } from '../LookupResolver' import { getOverlayHostReputationTracker } from '../HostReputationTracker' import OverlayAdminTokenTemplate from '../OverlayAdminTokenTemplate' @@ -993,6 +994,42 @@ describe('LookupResolver dynamic discovery', () => { await second.return?.() }) + it('throws a resource-limit error when discovery exhausts the byte budget before any host is admitted', async () => { + const tracker = 'https://discovery-limit-tracker.example' + const lookup = jest.fn( + async ( + _url: string, + _question: unknown, + _timeout: unknown, + _signal?: AbortSignal, + options?: { consumeBytes?: (bytes: number) => void } + ) => { + options?.consumeBytes?.(4096) + return { type: 'output-list' as const, outputs: [] } + } + ) + const resolver = new LookupResolver({ + facilitator: { lookup } as any, + slapTrackers: [tracker] + }) + const pending = resolver.queryDetailed( + { service: 'ls_discovery_limit', query: {} }, + undefined, + { limits: { maxTotalBytes: 1024 } } + ) + pending.catch(() => { + /* asserted below */ + }) + + await jest.runAllTimersAsync() + + await expect(pending).rejects.toBeInstanceOf(LookupResourceLimitError) + await expect(pending).rejects.toMatchObject({ + name: 'LookupResourceLimitError', + limit: 'maxTotalBytes' + }) + }) + it('throws from query() when a deadline expires before any host is admitted', async () => { const tracker = 'https://deadline-miss-tracker.example' const lookup = jest.fn( From 7584d4e8c2c17288e8982155a399ef73efbf0705 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 15:40:22 -0500 Subject: [PATCH 131/168] fix(overlay): keep SPV verification on historical-tx submissions A merge on this branch widened the submit() SPV guard from `mode !== 'historical-tx-no-spv'` to `mode === 'current-tx'`. That removed `tx.verify(this.chainTracker)` from every `historical-tx` submission, not just from the BASM admission path that needed it. `historical-tx` is a public submission mode, so any caller could get an unverified transaction applied to a topic. Root cause: the guard was widened to work around a symptom. BASM admission proves inclusion independently (`chainTracker.isValidRootForHeight` plus the canonical-header binding) and must not re-apply `MerklePath.verify`'s coinbase 100-block spendability rule, but the correct lever for that is the existing `historical-tx-no-spv` mode on the BASM call site, not a relaxed global guard. Restores the original guard and switches the BASM admission in `fetchBASMMissingTransactions` to `historical-tx-no-spv`. The other effects of that mode are correct here: propagation is skipped (already the case for `historical-tx`), and although the broadcast block is entered, every BASM-admitted transaction carries its extracted Merkle path, so `broadcastAcceptedSubmission` synthesises a local success and never reaches the network. Topic managers now observe `historical-tx-no-spv` for BASM admissions; no bundled manager branches on the mode. Tests: the coinbase fixture no longer mocks `Engine.submit`, so the young offset-0 proof is admitted through the production path, and a new test proves a GASP-style `historical-tx` submission with an invalid proof is still rejected. Co-Authored-By: Claude Fable 5.1 --- packages/overlays/overlay/README.md | 8 +- packages/overlays/overlay/src/Engine.ts | 17 ++-- .../src/__tests/BASMReconciliation.test.ts | 80 +++++++++++++++++-- 3 files changed, 91 insertions(+), 14 deletions(-) diff --git a/packages/overlays/overlay/README.md b/packages/overlays/overlay/README.md index 3e753aca6..3f9bbef58 100644 --- a/packages/overlays/overlay/README.md +++ b/packages/overlays/overlay/README.md @@ -199,8 +199,12 @@ submission. Claimed admitted-list indices are bound to the compound path wheneve a remote list is used as evidence, including when every remote txid is already local. Inclusion uses the chain tracker root/height check rather than `MerklePath.verify`, which also enforces coinbase 100-block spendability. -Historical mode still applies the local TopicManager and suppresses broadcast -and propagation. Automatic BASM sync remains disabled by default. +Because inclusion is proven independently, admission submits in the +`historical-tx-no-spv` mode so `Transaction.verify` does not re-apply that +coinbase rule; the public `historical-tx` mode keeps full SPV verification. +Admission still applies the local TopicManager, and because every admitted +transaction carries its extracted Merkle path, neither network broadcast nor +overlay propagation occurs. Automatic BASM sync remains disabled by default. This is bounded protocol hardening, not durable recovery. An empty local node whose topic genesis precedes the recent bootstrap window now refuses the diff --git a/packages/overlays/overlay/src/Engine.ts b/packages/overlays/overlay/src/Engine.ts index b79013b10..9f9982ce4 100644 --- a/packages/overlays/overlay/src/Engine.ts +++ b/packages/overlays/overlay/src/Engine.ts @@ -1071,12 +1071,11 @@ export class Engine { const txid = tx.id('hex') this.startTime(`submit_${txid}`) - // BASM reconciliation has already validated historical-tx inclusion with - // the compound Merkle proof and canonical header. Running Transaction.verify - // here would incorrectly apply coinbase maturity to an admitted historical - // transaction, even though BASM only proves inclusion. Current submissions - // still require the normal SPV/Bitcoin validity check. - if (mode === 'current-tx') { + // Every submission is SPV-verified except 'historical-tx-no-spv', which is + // reserved for callers that have already proven inclusion independently + // (GASP graph finalization and BASM reconciliation). 'historical-tx' is a + // public submission mode, so it keeps the full SPV check. + if (mode !== 'historical-tx-no-spv') { this.startTime(`chainTracker_${txid.substring(0, 10)}`) const txValid = await tx.verify(this.chainTracker) if (!txValid) throw new Error('Unable to verify SPV information.') @@ -1938,11 +1937,15 @@ export class Engine { const commitHeader = await this.requireCanonicalBASMAnchor(anchor, proofRoot) requireBASM(commitHeader.blockTransactionCount === proofHeader.blockTransactionCount, 'BASM canonical block transaction count changed before admission') // Apply in the independently checked block order, regardless of raw response order. + // Admit with 'historical-tx-no-spv': inclusion is already proven above by + // isValidRootForHeight plus the canonical-header binding. Re-running + // Transaction.verify would apply MerklePath.verify's coinbase 100-block + // spendability rule and reject an admitted coinbase from a recent block. const transactionById = new Map(transactions.map(tx => [tx.id('hex'), tx])) for (const txid of txids) { const tx = transactionById.get(txid) requireBASM(tx !== undefined, 'BASM raw response omits a requested transaction') - await this.submit({ beef: tx.toBEEF(), topics: [topic] }, undefined, 'historical-tx') + await this.submit({ beef: tx.toBEEF(), topics: [topic] }, undefined, 'historical-tx-no-spv') } const finalHeader = await this.requireCanonicalBASMAnchor(anchor, proofRoot) requireBASM(finalHeader.blockTransactionCount === proofHeader.blockTransactionCount, 'BASM canonical block transaction count changed during admission') diff --git a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts index 55d581791..358fc8b88 100644 --- a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts +++ b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts @@ -127,7 +127,7 @@ describe('BASM reconciliation evidence binding', () => { ).toEqual([f.ids[1], f.ids[3]]) for (const [, callback, mode] of f.submit.mock.calls) { expect(callback).toBeUndefined() - expect(mode).toBe('historical-tx') + expect(mode).toBe('historical-tx-no-spv') } expect(f.tracker.isValidRootForHeight).toHaveBeenCalledTimes(1) expect(f.tracker.isValidRootForHeight).toHaveBeenCalledWith(f.root, blockHeight) @@ -416,14 +416,42 @@ describe('BASM reconciliation evidence binding', () => { transactions: [{ txid, rawTx: coinbase.toHex() }], missing: [] } + const manager = { + identifyAdmissibleOutputs: jest.fn(async () => ({ outputsToAdmit: [0], coinsToRetain: [] })), + getDocumentation: jest.fn(async () => ''), + getMetaData: jest.fn(async () => ({ name: 'test', shortDescription: 'test' })) + } + f.engine.managers[topic] = manager + Object.assign(f.storage, { + doesAppliedTransactionExist: jest.fn(async () => false), + findOutput: jest.fn(async () => null), + insertOutput: jest.fn(async () => undefined), + insertAppliedTransaction: jest.fn(async () => undefined), + upsertTransactionRecord: jest.fn(async () => undefined) + }) + // Exercise the production submit path: the young coinbase must survive it. + f.submit.mockRestore() + const submit = jest.spyOn(f.engine, 'submit') const verify = jest.spyOn(MerklePath.prototype, 'verify') const [report] = await f.engine.startBASMSync() expect(report.status).toBe('advanced') expect(report.fetchedTxCount).toBe(1) expect(verify).not.toHaveBeenCalled() expect(f.tracker.isValidRootForHeight).toHaveBeenCalledWith(txid, blockHeight) - expect(f.submit).toHaveBeenCalledTimes(1) - expect(f.tracker.currentHeight).not.toHaveBeenCalled() + expect(submit).toHaveBeenCalledTimes(1) + expect(submit.mock.calls[0][2]).toBe('historical-tx-no-spv') + expect(manager.identifyAdmissibleOutputs).toHaveBeenCalledWith( + expect.anything(), + [], + undefined, + 'historical-tx-no-spv' + ) + expect(f.storage.insertAppliedTransaction).toHaveBeenCalledWith( + expect.objectContaining({ txid, topic, blockHeight, blockIndex: 0, proven: true }) + ) + // The only chain-height read is the firstSeen bookkeeping in + // buildAppliedTransactionRecord; no coinbase spendability check ran. + expect(f.tracker.currentHeight).toHaveBeenCalledTimes(1) }) it('admits a BASM-proven historical coinbase through the real submit path', async () => { @@ -451,15 +479,57 @@ describe('BASM reconciliation evidence binding', () => { }) f.submit.mockRestore() - await expect(f.engine.submit({ beef, topics: [topic] }, undefined, 'historical-tx')).resolves.toEqual({ + await expect( + f.engine.submit({ beef, topics: [topic] }, undefined, 'historical-tx-no-spv') + ).resolves.toEqual({ [topic]: { outputsToAdmit: [0], coinsToRetain: [], coinsRemoved: [] } }) - expect(manager.identifyAdmissibleOutputs).toHaveBeenCalledWith(beef, [], undefined, 'historical-tx') + expect(manager.identifyAdmissibleOutputs).toHaveBeenCalledWith( + beef, + [], + undefined, + 'historical-tx-no-spv' + ) expect(f.storage.insertAppliedTransaction).toHaveBeenCalledWith( expect.objectContaining({ txid, topic, proven: false }) ) }) + it('still runs SPV verification for a GASP-style historical-tx submission', async () => { + const f = fixture() + const proven = f.transactions[1] + // A proof whose root is not the canonical root for this height: the chain + // tracker rejects it, so MerklePath.verify fails and submit must throw. + proven.merklePath = new MerklePath(blockHeight, [ + [ + { offset: 0, hash: f.ids[0], txid: true }, + { offset: 1, hash: f.ids[1], txid: true } + ] + ]) + const beef = proven.toBEEF() + const manager = { + identifyAdmissibleOutputs: jest.fn(async () => ({ outputsToAdmit: [0], coinsToRetain: [] })), + getDocumentation: jest.fn(async () => ''), + getMetaData: jest.fn(async () => ({ name: 'test', shortDescription: 'test' })) + } + f.engine.managers[topic] = manager + Object.assign(f.storage, { + doesAppliedTransactionExist: jest.fn(async () => false), + findOutput: jest.fn(async () => null), + insertOutput: jest.fn(async () => undefined), + insertAppliedTransaction: jest.fn(async () => undefined), + upsertTransactionRecord: jest.fn(async () => undefined) + }) + f.submit.mockRestore() + + await expect( + f.engine.submit({ beef, topics: [topic] }, undefined, 'historical-tx') + ).rejects.toThrow(/Invalid merkle path/) + expect(f.tracker.isValidRootForHeight).toHaveBeenCalled() + expect(manager.identifyAdmissibleOutputs).not.toHaveBeenCalled() + expect(f.storage.insertAppliedTransaction).not.toHaveBeenCalled() + }) + it('reports a finite proof request limit for a block above 1000 admissions (B02 chunking required)', async () => { const f = fixture() const admitted = Array.from({ length: 1001 }, (_, blockIndex) => ({ From e87ef2e0c0017364eae4ba90ec0e69aa85de48bb Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 15:42:05 -0500 Subject: [PATCH 132/168] fix(sdk): refuse redirects on overlay lookup and SLAP discovery requests performLookupRequest built its RequestInit without a redirect field, so fetch defaulted to 'follow'. normalizeLookupHost and the https: guard in lookup() validate the advertised URL only; neither runs again on the final URL. A SLAP-advertised HTTPS host could therefore answer the lookup POST with a 307/308 and have the serialized stringifyBRC100({ service, query }) body re-sent to http:, loopback or a link-local address. This PR queries every eligible advertised host, so the hop sat on the hot path for untrusted SLAP domains, and the SLAP tracker discovery requests reach the network through this same method. Set redirect: 'error' so the transport fails closed. fetch rejects, and the existing failure path records the advertised host as an ordinary availability failure instead of crashing the query or following the hop. Co-Authored-By: Claude Fable 5.1 --- packages/sdk/CHANGELOG.md | 6 ++ .../sdk/src/overlay-tools/LookupResolver.ts | 8 +++ .../__tests/LookupResolver.additional.test.ts | 55 +++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index fb724536d..98b28c465 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -398,6 +398,12 @@ All notable changes to this project will be documented in this file. The format ### Security +- `HTTPSOverlayLookupFacilitator` now issues lookup and SLAP tracker discovery + requests with `redirect: 'error'`. A SLAP-advertised host can no longer + redirect the serialized lookup body to an origin that the advertised-host + scheme and credential checks never saw, such as `http:`, loopback, or + link-local. A redirected response is recorded as an ordinary availability + failure for the advertised host. - Treat cryptographic verification as successful only when it returns an affirmative result: `GlobalKVStore` rejects forged controller-signed overlay values, and `IdentityClient` refuses to publish signature-invalid identity diff --git a/packages/sdk/src/overlay-tools/LookupResolver.ts b/packages/sdk/src/overlay-tools/LookupResolver.ts index a5ad0bb30..ba93f4ca8 100644 --- a/packages/sdk/src/overlay-tools/LookupResolver.ts +++ b/packages/sdk/src/overlay-tools/LookupResolver.ts @@ -563,6 +563,14 @@ export class HTTPSOverlayLookupFacilitator implements OverlayLookupFacilitator { 'X-Aggregation': 'yes' }, body: stringifyBRC100({ service: question.service, query: question.query }), + // normalizeLookupHost and the https: guard above validate the advertised + // URL only. A followed 307/308 would carry the serialized query body to + // an origin neither check ever saw, so an untrusted SLAP host could + // redirect a lookup (or a tracker discovery request, which uses this + // same path) to http:, loopback or link-local. Fail closed instead: the + // rejection is recorded as an ordinary availability failure for the + // advertised host. + redirect: 'error', signal } const response: Response = await this.fetchClient(`${url}/lookup`, fco) diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts index 358dbba5f..3a782138a 100644 --- a/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.additional.test.ts @@ -489,6 +489,61 @@ describe('LookupResolver – additional coverage', () => { expect(result).toEqual({ type: 'output-list', outputs: [] }) }) + it('refuses to follow a redirect away from the advertised lookup host', async () => { + const mockFetch = jest + .fn() + .mockResolvedValue(jsonResponse({ type: 'output-list', outputs: [] })) + const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, false) + await facilitator.lookup('https://advertised.example', { service: 'ls_test', query: {} }) + expect(mockFetch).toHaveBeenCalledWith( + 'https://advertised.example/lookup', + expect.objectContaining({ redirect: 'error' }) + ) + }) + + it('treats a rejected redirect as a host failure rather than a crash', async () => { + // fetch rejects with a TypeError when redirect: 'error' meets a 307/308. + const mockFetch = jest.fn().mockRejectedValue(new TypeError('unexpected redirect')) + const resolver = new LookupResolver({ + facilitator: new HTTPSOverlayLookupFacilitator(mockFetch, false), + hostOverrides: { ls_redirect: ['https://redirecting.example'] } + }) + + const result = await resolver.queryDetailed({ service: 'ls_redirect', query: {} }) + + expect(result.answer).toEqual({ type: 'output-list', outputs: [] }) + expect(result.progress).toMatchObject({ + hostCount: 1, + failedHosts: 1, + successfulHosts: 0, + rejectedHosts: 0, + terminalReason: 'settled' + }) + expect(mockFetch).toHaveBeenCalledTimes(1) + expect(mockFetch).toHaveBeenCalledWith( + 'https://redirecting.example/lookup', + expect.objectContaining({ redirect: 'error' }) + ) + }) + + it('refuses to follow a redirect on SLAP tracker discovery requests', async () => { + const mockFetch = jest + .fn() + .mockResolvedValue(jsonResponse({ type: 'output-list', outputs: [] })) + const resolver = new LookupResolver({ + facilitator: new HTTPSOverlayLookupFacilitator(mockFetch, false), + slapTrackers: ['https://tracker.example'] + }) + + await expect(resolver.query({ service: 'ls_redirect_tracker', query: {} })).rejects.toThrow( + 'No competent mainnet hosts found' + ) + expect(mockFetch).toHaveBeenCalledWith( + 'https://tracker.example/lookup', + expect.objectContaining({ redirect: 'error' }) + ) + }) + it('handles HTTP error responses by throwing', async () => { const mockFetch = jest.fn().mockResolvedValue(jsonResponse({}, 503)) const facilitator = new HTTPSOverlayLookupFacilitator(mockFetch, true) From 55d64a2510d00f2cdfee6d57dd468c7070ec0ad5 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 15:54:46 -0500 Subject: [PATCH 133/168] test(overlay): gate Go interop solely on BASM_GO_OVERLAY_SERVICES `resolveGoWorktree()` tried the `BASM_GO_OVERLAY_SERVICES` environment variable and then fell back to the literal path `/Users/personal/git/go/worktrees/go-overlay-services-basm`. That fallback made the suite run on exactly one workstation and `describe.skip` everywhere else, including CI, so the B01 protocol was never actually spoken by any job that claimed to cover it, and the skip was silent. Root cause: the checkout location was treated as a discovery problem with a convenient default instead of an explicit input. Resolution is now the environment variable alone. When it is unset the suite is skipped under a title that names the variable; when it is set but does not resolve to a go-overlay-services checkout the resolver throws, so a job configured for Go interop fails closed rather than quietly skipping. Tests: a new BASMGoInteropGating suite asserts the gate from the interop source, which is the only machine-independent way to catch this class of regression - a behavioural assertion agrees with a workstation fallback on every machine that lacks that path, which is precisely where the fallback hides. The fixtures README now states that Go interop does not run by default and records the fixture provenance as a repository-relative path. Verified locally: with BASM_GO_OVERLAY_SERVICES=/Users/personal/git/go/worktrees/go-overlay-services-basm the interop suite builds the Go host and passes (1/1); with a non-checkout path the suite fails to run with the explanatory error; unset, it skips. Co-Authored-By: Claude Fable 5.1 --- .../overlay/src/__tests/BASMGoInterop.test.ts | 39 ++++++++++++------- .../src/__tests/BASMGoInteropGating.test.ts | 35 +++++++++++++++++ .../overlay/src/__tests/fixtures/README.md | 10 ++++- 3 files changed, 68 insertions(+), 16 deletions(-) create mode 100644 packages/overlays/overlay/src/__tests/BASMGoInteropGating.test.ts diff --git a/packages/overlays/overlay/src/__tests/BASMGoInterop.test.ts b/packages/overlays/overlay/src/__tests/BASMGoInterop.test.ts index 42169f66e..65ac52073 100644 --- a/packages/overlays/overlay/src/__tests/BASMGoInterop.test.ts +++ b/packages/overlays/overlay/src/__tests/BASMGoInterop.test.ts @@ -29,21 +29,28 @@ function independentTac(previous: string, blockHash: string, root: string): stri return Buffer.from(createHash('sha256').update(first).digest()).reverse().toString('hex') } +const GO_CHECKOUT_MARKERS = [ + 'pkg/server/server_http_basm_interop_test.go', + 'pkg/core/engine/basm-read-service.go' +] + +/** + * Resolves the go-overlay-services checkout to run interop against. The only + * source is BASM_GO_OVERLAY_SERVICES: no workstation path is assumed, so this + * suite never silently depends on one machine's layout. When the variable is + * set but does not point at a usable checkout we fail rather than skip, so a + * job that claims Go interop cannot pass without running it. + */ function resolveGoWorktree(): string | undefined { - const candidates = [ - process.env.BASM_GO_OVERLAY_SERVICES, - '/Users/personal/git/go/worktrees/go-overlay-services-basm' - ] - for (const candidate of candidates) { - if ( - candidate !== undefined && - existsSync(join(candidate, 'pkg/server/server_http_basm_interop_test.go')) && - existsSync(join(candidate, 'pkg/core/engine/basm-read-service.go')) - ) { - return candidate - } + const candidate = process.env.BASM_GO_OVERLAY_SERVICES + if (candidate === undefined || candidate === '') return undefined + const missing = GO_CHECKOUT_MARKERS.filter(marker => !existsSync(join(candidate, marker))) + if (missing.length > 0) { + throw new Error( + `BASM_GO_OVERLAY_SERVICES is set to "${candidate}" but that is not a go-overlay-services checkout (missing: ${missing.join(', ')})` + ) } - return undefined + return candidate } async function waitForUrls( @@ -140,8 +147,12 @@ function startGoHost(goRoot: string): ChildProcessWithoutNullStreams { const goRoot = resolveGoWorktree() const describeInterop = goRoot === undefined ? describe.skip : describe +const suiteName = + goRoot === undefined + ? 'BASMRemote localhost interop against Go read/serving (skipped: set BASM_GO_OVERLAY_SERVICES to a go-overlay-services checkout to run it)' + : 'BASMRemote localhost interop against Go read/serving' -describeInterop('BASMRemote localhost interop against Go read/serving', () => { +describeInterop(suiteName, () => { let child: ChildProcessWithoutNullStreams let readyURL = '' let unsupportedURL = '' diff --git a/packages/overlays/overlay/src/__tests/BASMGoInteropGating.test.ts b/packages/overlays/overlay/src/__tests/BASMGoInteropGating.test.ts new file mode 100644 index 000000000..e5746a32c --- /dev/null +++ b/packages/overlays/overlay/src/__tests/BASMGoInteropGating.test.ts @@ -0,0 +1,35 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' + +const INTEROP_SOURCE = join(__dirname, 'BASMGoInterop.test.ts') + +/** + * The Go read/serving interop suite can only run where a go-overlay-services + * checkout exists, so it is gated. The gate itself has to be asserted from the + * source text: a behavioural test would agree with a workstation-path fallback + * on every machine that lacks that path, which is exactly the environment (CI) + * where such a fallback hides the fact that the suite never runs. + */ +describe('BASM Go interop gating', () => { + const source = readFileSync(INTEROP_SOURCE, 'utf8') + + it('does not hard-code a developer workstation checkout path', () => { + expect(source).not.toMatch(/\/(?:Users|home)\/[A-Za-z0-9._-]+\//) + }) + + it('resolves the Go checkout only from BASM_GO_OVERLAY_SERVICES', () => { + const resolver = /\nfunction resolveGoWorktree\(\)[\s\S]*?\n}\n/.exec(source)?.[0] + expect(resolver).toBeDefined() + expect(resolver).toContain('process.env.BASM_GO_OVERLAY_SERVICES') + expect(resolver?.match(/process\.env\.[A-Za-z_]+/g)).toEqual([ + 'process.env.BASM_GO_OVERLAY_SERVICES' + ]) + }) + + it('names the environment variable in the skipped suite title', () => { + const skipTitle = /goRoot === undefined\s*\?\s*'([^']*)'/.exec(source)?.[1] + expect(skipTitle).toBeDefined() + expect(skipTitle).toContain('skipped') + expect(skipTitle).toContain('BASM_GO_OVERLAY_SERVICES') + }) +}) diff --git a/packages/overlays/overlay/src/__tests/fixtures/README.md b/packages/overlays/overlay/src/__tests/fixtures/README.md index b5d4fb03e..48ddcce21 100644 --- a/packages/overlays/overlay/src/__tests/fixtures/README.md +++ b/packages/overlays/overlay/src/__tests/fixtures/README.md @@ -1,9 +1,9 @@ # Independent BRC-136 fixture brc136-independent.json is copied byte-for-byte from the Go BASM -foundation fixture: +foundation fixture in the `go-overlay-services` repository: -/Users/personal/git/go/worktrees/go-overlay-services-basm/pkg/core/basm/testdata/vectors.json +pkg/core/basm/testdata/vectors.json Source commit: d99216814a4b9dca5f9f4d04a602ef2d48bdc4a7 Go worktree revision used for inventory: faaf69d372fd5e9974eaadbae9b8e26d761f0c86 @@ -25,5 +25,11 @@ values as authoritative without that independent check. module. It replaces onto the Go overlay-services worktree and must not modify that tree's uncommitted S04 files. +The Go interop suite (`BASMGoInterop.test.ts`) does not run by default: it is +skipped unless `BASM_GO_OVERLAY_SERVICES` points at a `go-overlay-services` +checkout, and no workstation path is assumed. When the variable is set but does +not resolve to such a checkout the suite fails rather than skips, so a job that +claims Go interop cannot pass without running it. + The temporary interop server binds and advertises only `127.0.0.1`; its HTTP transport never exposes the fixture on an external network interface. From 1eaf09596a4e8396133bcc8165cb3e293721a59b Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 15:55:47 -0500 Subject: [PATCH 134/168] fix(overlay-mongo): serialize pin re-add expiry with GC reclaim addReference's idempotent re-add branch only re-validated the payload was still ready; it never touched the existing (ownerKind, ownerId, slot) pin row's expiresAt. Extending a pin therefore silently kept the stale expiry, and claimGarbage counted an already-expired pin as not-live via $expr {$gt:['$expiresAt','$$NOW']} but never deleted that expired pin document. The unique slot row survived expiry and kept short-circuiting future addReference calls on that slot, so a caller could be told content stayed pinned while GC moved the payload ready -> deleting underneath it. Add refreshPinExpiry: called after the existing {_id: payloadId, state: 'ready'} CAS matches (so it stays serialized against a concurrent GC claim in the same transaction), it reactivates an expired pin with the caller's new expiry, allows extending a still- live pin to a later expiry, and rejects an attempt to shorten a live pin's expiry rather than silently ignoring it. claimGarbage now deletes every expired pin reference row for the payload in the same transaction as its ready -> deleting claim, so a stale slot cannot outlive the payload it named. Document the re-add/extend/reactivate/shorten rule in specs/overlay/mongo-v1.md next to the existing pin semantics. Regression tests (mongodb-memory-server replica fixture) added to src/__tests/mongo/MongoPayloadStore.test.ts: - re-adding a pin slot extends its expiry but never shortens it silently - re-adding an expired pin slot reactivates it and makes the payload live again - claimGarbage deletes the expired pin reference row inside the same transaction as its claim All three were confirmed to fail against the pre-fix source for the reason described above (verified by temporarily reverting this file and re-running the suite) before the fix was applied. Co-Authored-By: Claude Fable 5.1 --- .../__tests/mongo/MongoPayloadStore.test.ts | 128 ++++++++++++++++++ .../src/storage/mongo/MongoPayloadStore.ts | 44 +++++- specs/overlay/mongo-v1.md | 8 ++ 3 files changed, 179 insertions(+), 1 deletion(-) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts index ecdc5d069..35d8927e0 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts @@ -314,6 +314,134 @@ describe('MongoPayloadStore', () => { await session.endSession() }) + test('re-adding a pin slot extends its expiry but never shortens it silently', async () => { + const content = Buffer.from('pin-extend') + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const basePin = { + scope: fixture.scope, + payload, + ownerKind: 'pin' as const, + ownerId: 'pin-extend', + slot: '0' + } + const firstExpiry = new Date(Date.now() + 60_000) + const session = fixture.client.startSession() + await session.withTransaction(async () => { + await store.addReference(session, { ...basePin, expiresAt: firstExpiry }) + }) + const laterExpiry = new Date(Date.now() + 120_000) + await session.withTransaction(async () => { + await store.addReference(session, { ...basePin, expiresAt: laterExpiry }) + }) + expect( + ( + await fixture.db + .collection('overlay_payload_references') + .findOne({ ownerId: 'pin-extend' }) + )?.expiresAt + ).toEqual(laterExpiry) + const earlierExpiry = new Date(Date.now() + 90_000) + await expect( + session.withTransaction(async () => { + await store.addReference(session, { ...basePin, expiresAt: earlierExpiry }) + }) + ).rejects.toThrow('must not shorten') + expect( + ( + await fixture.db + .collection('overlay_payload_references') + .findOne({ ownerId: 'pin-extend' }) + )?.expiresAt + ).toEqual(laterExpiry) + await session.endSession() + }) + + test('re-adding an expired pin slot reactivates it and makes the payload live again', async () => { + const content = Buffer.from('pin-reactivate') + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const pin = { + scope: fixture.scope, + payload, + ownerKind: 'pin' as const, + ownerId: 'pin-reactivate', + slot: '0', + expiresAt: new Date(Date.now() + 60_000) + } + const session = fixture.client.startSession() + await session.withTransaction(async () => { + await store.addReference(session, pin) + }) + await fixture.db.collection('overlay_payload_references').updateOne({ ownerId: 'pin-reactivate' }, [ + { + $set: { expiresAt: { $dateSubtract: { startDate: '$$NOW', unit: 'second', amount: 1 } } } + } + ]) + const revived = { ...pin, expiresAt: new Date(Date.now() + 60_000) } + await session.withTransaction(async () => { + await store.addReference(session, revived) + }) + await session.withTransaction(async () => { + expect(await store.claimGarbage(session, payload)).toBe(false) + }) + expect( + ( + await fixture.db + .collection('overlay_payload_references') + .findOne({ ownerId: 'pin-reactivate' }) + )?.expiresAt + ).toEqual(revived.expiresAt) + await session.endSession() + }) + + test('claimGarbage deletes the expired pin reference row inside the same transaction as its claim', async () => { + const content = Buffer.from('pin-gc-cleanup') + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const pin = { + scope: fixture.scope, + payload, + ownerKind: 'pin' as const, + ownerId: 'pin-gc-cleanup', + slot: '0', + expiresAt: new Date(Date.now() + 60_000) + } + const session = fixture.client.startSession() + await session.withTransaction(async () => { + await store.addReference(session, pin) + }) + await fixture.db.collection('overlay_payload_references').updateOne({ ownerId: 'pin-gc-cleanup' }, [ + { + $set: { expiresAt: { $dateSubtract: { startDate: '$$NOW', unit: 'second', amount: 1 } } } + } + ]) + await session.withTransaction(async () => { + expect(await store.claimGarbage(session, payload)).toBe(true) + }) + expect( + await fixture.db + .collection('overlay_payload_references') + .countDocuments({ ownerId: 'pin-gc-cleanup' }) + ).toBe(0) + await session.endSession() + }) + test('uses majority journaled writes even when the caller client defaults to w:1', async () => { const weak = await fixture.connect({ monitorCommands: true, writeConcern: { w: 1 } }) const concerns: unknown[] = [] diff --git a/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts b/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts index 73835ef66..1ee0d309f 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts @@ -227,13 +227,16 @@ export class MongoPayloadStore { if (existing.payloadId !== payloadId) throw new Error('Mongo payload reference slot already names different content') // An idempotent reference still needs a live ready payload. The row may - // have been reclaimed since the reference was written. + // have been reclaimed since the reference was written. This CAS also + // serializes the pin refresh below against a concurrent GC claim. const guarded = await payloads.updateOne( { _id: payloadId, state: 'ready' }, { $set: { updatedAt: this.now() } }, { session, timeoutMS: operation.timeoutMS } ) if (guarded.matchedCount !== 1) throw new Error('Mongo payload is not ready for reference') + if (reference.ownerKind === 'pin') + await this.refreshPinExpiry(refs, refId, reference, existing, session, operation) return } // This conditional write is the guard; a snapshot read followed by an insert is unsafe. @@ -262,6 +265,37 @@ export class MongoPayloadStore { ) } + /** + * Refreshes an existing pin's expiry when the same slot is re-added: an + * expired pin is reactivated with the caller's new expiry, and a still-live + * pin may only be extended, never silently shortened. Called only after the + * caller's {_id: payloadId, state: 'ready'} CAS has already matched in this + * transaction, so this stays serialized against a concurrent GC claim. + */ + private async refreshPinExpiry( + refs: Collection, + refId: string, + reference: MongoPayloadReference, + existing: ReferenceDocument, + session: ClientSession, + operation: MongoPayloadOperationOptions + ): Promise { + const now = this.now() + // validateReference requires expiresAt for every pin reference. + const requested = reference.expiresAt as Date + const current = existing.expiresAt + const currentExpired = current === undefined || current <= now + if (!currentExpired && requested < current) + throw new Error('Mongo payload pin re-add must not shorten an unexpired expiry') + if (currentExpired || requested.getTime() !== current.getTime()) { + await refs.updateOne( + { _id: refId }, + { $set: { expiresAt: requested, updatedAt: now } }, + { session, timeoutMS: operation.timeoutMS } + ) + } + } + /** Must be called in the transaction that releases the owner record's payload obligation. */ public async releaseReference( session: ClientSession, @@ -378,6 +412,14 @@ export class MongoPayloadStore { { session, timeoutMS: operation.timeoutMS, returnDocument: 'after' } ) if (claimed === null) return false + // Every pin row still present here was already confirmed expired by the + // liveReferences count above; delete it in this same transaction so the + // unique slot cannot outlive the payload it named and short-circuit a + // future addReference on that slot. + await refs.deleteMany( + { payloadId, ownerKind: 'pin', $expr: { $lte: ['$expiresAt', '$$NOW'] } }, + { session, timeoutMS: operation.timeoutMS } + ) await this.options.hooks?.afterDeleteClaim?.() return true } diff --git a/specs/overlay/mongo-v1.md b/specs/overlay/mongo-v1.md index 93d0a0f32..ebeea19f0 100644 --- a/specs/overlay/mongo-v1.md +++ b/specs/overlay/mongo-v1.md @@ -86,6 +86,14 @@ GridFS deletion and `deleted` finalization resume separately and are safe to retry after a process failure. Pins are references and are the only reference kind allowed to expire. +Re-adding an existing pin slot is serialized under that same `ready`-row guard. +An expired pin is reactivated with the caller's new expiry; a still-live pin +may only be extended to a later expiry and never shortened silently — a +shortening request is rejected. Garbage collection deletes each expired pin's +reference row in the same transaction as its `ready` to `deleting` claim, so a +stale pin slot cannot outlive the payload it names and keep short-circuiting +future reference creation on that slot. + Payload and reference operations accept bounded timeout and cancellation controls where they operate in a caller's admission body. Neither payload publication nor collection runs verifier logic, network activity, uploads, or From b6ae7b7da651e5515fe0d78310153b30ba4c2bd8 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 15:56:15 -0500 Subject: [PATCH 135/168] docs(overlay): record the BASM admission submission mode Follow-up to 7584d4e8c, which moved BASM admission from the public `historical-tx` submission mode to `historical-tx-no-spv` so that restoring the SPV guard does not re-apply MerklePath.verify's coinbase 100-block spendability rule to an independently proven admission. The submission mode is passed through to `TopicManager.identifyAdmissibleOutputs`, so it is observable by every topic manager that branches on it. The release prose described the inclusion rule but not the mode a manager now sees, which left implementors without the one externally visible consequence of the change. Adds that sentence to the @bsv/overlay release note and the package CHANGELOG entry, and regenerates docs/reference/package-api-migrations.md from the governance source. No code or behaviour change. Co-Authored-By: Claude Fable 5.1 --- docs/reference/package-api-migrations.md | 4 ++-- governance/package-release-notes.json | 2 +- packages/overlays/overlay/CHANGELOG.md | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 35cb5feaf..0c370b6bb 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -42,7 +42,7 @@ and clean-consumer tests remain the executable type authority. | `@bsv/gasp` | `1.3.1` | `1.3.6` | patch | [API and usage](../packages/overlays/gasp.md) | No consumer migration is required; existing constructor calls, imports, synchronization behavior, and wire semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/lch` | `0.0.0` | `0.1.0` | minor | [API and usage](../packages/content/lch.md) | No consumer migration is required; this is the first release of a new additive package. Applications must give quote() the verified Asset encryption descriptor and selected Offer key-delivery mechanism, keep createPayment behind explicit wallet authorization, persist the funded transaction and every partial settlement proof through recovery, retry with that same transaction, distinguish finalized from accepted evidence, fail closed on unknown settlement or evidence profiles, and supply an address-pinning connector for DNS endpoints. Payees should select receipt-complete unless they explicitly accept authorized-output provider, privacy, and pre-internalization key-release tradeoffs. Distributors must retain THIRD_PARTY_NOTICES.md with the package; the new profile adds no dependency. Published BRC-170 remains authoritative if the implementation and standard differ. | | `@bsv/message-box-client` | `2.4.0` | `2.5.1` | minor | [API and usage](../packages/messaging/message-box-client.md) | No API migration is required. socketOptions is optional and nothing is forwarded by default, so socket transport negotiation and every HTTP code path are unchanged for existing callers; wallet and originator stay client-owned and cannot be overridden through it. managerOptions excludes autoConnect and retries: JavaScript callers receive an immediate error for autoConnect: false or nonzero message retries because the live socket starts immediately and AuthSocket does not send Socket.IO acknowledgements. Connection reconnection options remain supported. Upgrade @bsv/sdk and @bsv/message-box-client together; historical number-array wallets, current Uint8Array substrates, and already-pending numeric-key messages interoperate through the same portable transaction form. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. No migration is required for the bundled payment optimization. Module consumers can use SDK 2.5.0 or later to enable the same optional extension; existing compatible SDK peer versions remain supported. Recipients must advertise already-validated transaction IDs through x-bsv-payment-known-txids, an optional SDK extension rather than a standardized BRC-105 header; services that omit it retain existing payment behavior. Existing return shapes and payment envelopes remain unchanged. Failed or incomplete notification payments stay queued. Resolve uncertain refund-send outcomes before retrying; this patch adds ordering checks, not an exactly-once refund journal. listMessages/listMessagesLite envelope behavior and basket-insertion semantics remain tracked in issue #503. | -| `@bsv/overlay` | `2.2.1` | `2.5.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. | +| `@bsv/overlay` | `2.2.1` | `2.5.0` | minor | [API and usage](../packages/overlays/overlay.md) | Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. BASM admission therefore submits in the historical-tx-no-spv mode, so topic managers that branch on the submission mode observe historical-tx-no-spv rather than historical-tx for BASM-admitted transactions. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. | | `@bsv/overlay-discovery-services` | `2.1.1` | `2.2.3` | minor | [API and usage](../packages/overlays/overlay-discovery-services.md) | Existing mainnet and testnet advertisers are unchanged. TTN operators pass chain ttn and provision the staging storage and overlay endpoints before advertising. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update. | | `@bsv/overlay-express` | `2.5.0` | `2.7.0` | minor | [API and usage](../packages/overlays/overlay-express.md) | Existing mainnet and testnet servers are unchanged. TTN servers call configureNetwork('ttn'), configureArcade with the TTN endpoint, and configureChaintracks or configureChainTracker before engine initialization. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. Valid legacy JSON shapes, numeric-string heights, public CORS, topic-free raw transactions, configured limits and synchronization defaults remain supported. No storage migration is required. Malformed/coercive inputs are rejected and missing BASM engine/storage capabilities return HTTP 400 with code BASM_UNSUPPORTED. Existing header providers remain valid without a blockTransactionCount field. | | `@bsv/overlay-topics` | `1.6.10` | `1.8.2` | minor | [API and usage](../packages/overlays/overlay-topics.md) | Existing topic and lookup identifiers remain unchanged. Production UMP overlays must give UMPTopicManager and the UMP lookup service Mongo-backed stores that use the same database, then roll out before updated wallet clients; the no-argument manager is bounded but intended only for isolated single-process use. The reservation and bootstrap-marker collections are additive and initialize from currently indexed UMP UTXOs; take a MongoDB backup before rollout. Legacy ambiguous rows remain visible and can be resolved with WAB pinning rather than deleted. Valid uora-anchor-v3 outputs retain their bytes, signatures, and admission result. Readers now reject nonconforming key, tail, and text shapes. Coordinate reader upgrades across nodes serving tm_uora_dpp and audit any previously indexed nonconforming outputs before rebuilding that topic. No wire prefix, export, or persisted schema changes are required. Mandala custom state adapters must implement isAdminOutpoint against admitted per-asset admin history and retain matching owner rows during admission. Registration must omit assetId or use an empty string. Back up and audit historical admin and ownership records before replay; missing records must be restored from verified admission evidence. Valid wire fields and encodings are unchanged, and no collection migration is needed. Wallets that reveal input linkage must reveal the spender's own child key. No migration is required for this dependency candidate update; default storage selection and existing runtime behavior remain unchanged. No migration is required for this dependency candidate update. | @@ -278,7 +278,7 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay.md](../packages/overlays/overlay.md) - Source: [packages/overlays/overlay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay) - Release note: Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Claimed admitted-list indices are bound to the compound path even when every remote txid is already local, and inclusion uses chain-tracker root/height rather than coinbase maturity. -- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. +- Migration: Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. BASM admission therefore submits in the historical-tx-no-spv mode, so topic managers that branch on the submission mode observe historical-tx-no-spv rather than historical-tx for BASM-admitted transactions. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index 91c7987e6..e9990915c 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -127,7 +127,7 @@ "publishedVersion": "2.2.1", "releaseType": "minor", "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Claimed admitted-list indices are bound to the compound path even when every remote txid is already local, and inclusion uses chain-tracker root/height rather than coinbase maturity.", - "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included." + "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Current Engine/Knex behavior is unchanged; no production adapter provides this atomic receipt yet. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No Engine activation, default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. BASM admission therefore submits in the historical-tx-no-spv mode, so topic managers that branch on the submission mode observe historical-tx-no-spv rather than historical-tx for BASM-admitted transactions. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included." }, { "name": "@bsv/overlay-discovery-services", diff --git a/packages/overlays/overlay/CHANGELOG.md b/packages/overlays/overlay/CHANGELOG.md index d2ca3d3c9..1a4a22f96 100644 --- a/packages/overlays/overlay/CHANGELOG.md +++ b/packages/overlays/overlay/CHANGELOG.md @@ -31,7 +31,10 @@ All notable changes to this project will be documented in this file. The format position assurance in sync reports. Fixed default forward pages to fit the public server's 1,000-anchor limit. Claimed admitted-list indices are bound to the compound path even when every remote txid is already local. Inclusion uses - chain-tracker root/height rather than coinbase maturity. Historical sync + chain-tracker root/height rather than coinbase maturity, so BASM admission + submits in the `historical-tx-no-spv` mode and topic managers that branch on + the submission mode observe it for BASM-admitted transactions; the public + `historical-tx` mode keeps full SPV verification. Historical sync refuses an untrusted TAC prefix; durable bootstrap, chunked recovery and topic status remain pending. No storage migration or automatic sync activation is included. From a7a49962ad2ab69a2a2c35fee4bfa96546207fa9 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 15:57:06 -0500 Subject: [PATCH 136/168] test(overlay-mongo): cover the too-small declared length reclaim path The reviewer asked for a regression test on the reclaim path taken when publish fails because the caller declared a too-small byteLength: consumePayloadChunk rejects the stream once it exceeds the declared length, abandonUpload marks the payload row 'deleted', and a retry with the correct digest and length must reclaim that row through tryClaimUpload (which already recomputes byteLength from the new input when the prior state is 'deleted') and finish 'ready' with the corrected byteLength stored. This path was already implemented and covered indirectly (a sibling test manually sets state: 'deleted' with a wrong byteLength via direct DB mutation), but there was no test driving the actual publish/fail/ retry sequence end to end. This is coverage-only: the reclaim logic in tryClaimUpload predates this change and needed no fix, so there is no meaningful pre-fix state to fail against in this worktree; the new test passes against the current code, which is the expected outcome for confirming existing, correct behavior. Co-Authored-By: Claude Fable 5.1 --- .../__tests/mongo/MongoPayloadStore.test.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts index 35d8927e0..9b2a906a7 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts @@ -610,6 +610,33 @@ describe('MongoPayloadStore', () => { expect(BigInt((await payloads.findOne({ _id: row?._id }))?.byteLength.toString() ?? '0')).toBe(BigInt(content.byteLength)) }) + test('recovers a too-small declared length by reclaiming the deleted row on retry with the correct length', async () => { + const content = Buffer.from('too-small-declared-length') + const hash = digest(content) + const payloads = fixture.db.collection('overlay_payloads') + await expect( + store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength - 1), + bytes: bytes(content) + }) + ).rejects.toThrow('exceeds declared length') + const failed = await payloads.findOne({ kind: 'outbox-data', digest: hash }) + expect(failed?.state).toBe('deleted') + await expect( + store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + ).resolves.toMatchObject({ digest: hash, byteLength: String(content.byteLength) }) + const recovered = await payloads.findOne({ _id: failed?._id }) + expect(recovered?.state).toBe('ready') + expect(BigInt(recovered?.byteLength.toString() ?? '0')).toBe(BigInt(content.byteLength)) + }) + test('does not reuse an existing reference after its payload was reclaimed', async () => { const content = Buffer.from('reclaimed-reference') const hash = digest(content) From 091f03e0d83a265f5d153acf48088e174ebea023 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 16:13:01 -0500 Subject: [PATCH 137/168] fix(sdk): re-queue displaced same-txid candidates instead of dropping them TransactionEvidenceCoordinator.run() shifts the next candidate off a job's queue before calling attempt(). If every concurrency slot is already in use (including slots still reserved by non-abortable, already-finished attempts; see the "does not free a non-abortable backend slot..." test), attempt() threw a plain TransactionEvidenceError('limit') and called the caller's `settled` callback. run()'s catch treated that exactly like a genuine verification failure: the candidate was gone (already shift()ed) and never pushed back, so a same-txid alternate candidate that had already been admitted to the job could be silently discarded without ever being tried, even though it was perfectly valid. Fix: attempt() now throws a dedicated ConcurrencyLimitSignal when the operation never started, and does not invoke `settled` (nothing was consumed, so no byte/candidate bookkeeping should be released). run()'s candidate loop catches that signal specifically, unshifts the candidate back onto job.candidates, marks the job not-running, and returns without looping or retrying itself. Re-admission is driven solely by pump() being invoked again when some other active attempt settles, so this cannot spin or busy-wait for a slot. Added a regression test that builds a genuine concurrency race: a non-abortable "ghost" attempt (cancelled while its script verification is still in flight) holds one of two slots, a job with two same-txid candidates (one invalid, one valid) takes the other, and a third waiting job is admitted into the slot freed by the first candidate's failure before the job's own retry can reclaim it. Before the fix this made the valid alternate candidate reject with 'limit'; the test fails for that reason on the old code and passes with the fix. Co-Authored-By: Claude Fable 5.1 --- .../TransactionEvidenceCoordinator.ts | 23 +++- .../TransactionEvidenceCoordinator.test.ts | 119 ++++++++++++++++++ 2 files changed, 140 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts b/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts index bdeb252c6..fb7a520ce 100644 --- a/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts +++ b/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts @@ -67,6 +67,15 @@ function outcome(error: unknown): TransactionEvidenceError { return evidenceError(error) } +/** + * Internal-only signal: this attempt was never started because every concurrency + * slot is in use (including slots still reserved by non-abortable, already-finished + * work). It is distinct from TransactionEvidenceError('limit') so a candidate that + * was merely displaced can be re-queued instead of treated as a tried-and-failed + * candidate. + */ +class ConcurrencyLimitSignal extends Error {} + /** * Bounded, process-local transaction evidence work sharing. This is independent of * lookup services, certificates and trust ratings. Positive reuse always checks @@ -364,6 +373,14 @@ export class TransactionEvidenceCoordinator { this.finish(job, positive) return } catch (error_) { + if (error_ instanceof ConcurrencyLimitSignal) { + // Never attempted: put it back rather than consuming it as a failure. + // Re-admission is driven only by an active attempt's own settle (via + // pump()); we do not loop or retry here, so this cannot spin. + job.candidates.unshift(candidate) + job.running = false + return + } error = outcome(error_) } } @@ -379,8 +396,10 @@ export class TransactionEvidenceCoordinator { settled?: () => void ): Promise { if (this.activeAttempts >= this.limits.concurrentTransactions) { - settled?.() - throw new TransactionEvidenceError('limit') + // The operation never started, so there is nothing for `settled` to release: + // any candidate bytes stay reserved and, for the candidate loop, the caller + // re-queues the candidate instead of discarding it. + throw new ConcurrencyLimitSignal() } this.activeAttempts++ const controller = new AbortController() diff --git a/packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts b/packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts index 5eed91b31..bfbfe484b 100644 --- a/packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts +++ b/packages/sdk/src/transaction/__tests/TransactionEvidenceCoordinator.test.ts @@ -265,6 +265,38 @@ async function twoAnchorFixture(): Promise<{ return { tracker, tx, evidence: tx.toBEEF(), roots } } +async function standaloneFixture( + keySeed: number +): Promise<{ tx: Transaction; root: string; evidence: number[] }> { + const key = new PrivateKey(keySeed) + const p2pkh = new P2PKH() + const source = new Transaction() + source.addInput({ + sourceTXID: '00'.repeat(32), + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + source.addOutput({ satoshis: 10, lockingScript: p2pkh.lock(key.toAddress()) }) + source.merklePath = new MerklePath(height, [ + [ + { offset: 0, hash: source.id('hex'), txid: true }, + { offset: 1, duplicate: true } + ] + ]) + const root = source.merklePath.computeRoot(source.id('hex')) + + const tx = new Transaction() + tx.addInput({ + sourceTransaction: source, + sourceOutputIndex: 0, + unlockingScriptTemplate: p2pkh.unlock(key) + }) + tx.addOutput({ satoshis: 4, lockingScript: p2pkh.lock(key.toAddress()) }) + tx.addOutput({ satoshis: 4, lockingScript: p2pkh.lock(key.toAddress()) }) + await tx.sign() + return { tx, root, evidence: tx.toBEEF() } +} + function verifyParamsInJavaScript(params: BdkVerifyScriptsParams): boolean { const sigHashCache = { hashOutputsSingle: new Map() } for (const [inputIndex, input] of params.tx.inputs.entries()) { @@ -800,6 +832,93 @@ describe('TransactionEvidenceCoordinator', () => { expect(calls).toBeGreaterThanOrEqual(2) }) + it('re-queues a same-txid alternate candidate displaced by a concurrency-limited attempt instead of losing it', async () => { + const ghost = await standaloneFixture(90_001) + const jobA = await standaloneFixture(90_002) + const jobD = await standaloneFixture(90_003) + + const tracker = new LocalChainTracker() + tracker.roots.add(ghost.root) + tracker.roots.add(jobD.root) + // jobA's original root is deliberately left out of tracker.roots so its + // first candidate fails and the coordinator must fall back to the second. + + const scriptGate = deferred() + const scriptEntered = deferred() + const verifier: BdkVerifierInterface = { + supportsMemoryLimit: true, + verifyScripts: async params => verifyParamsInJavaScript(params), + verifyScriptsBatch: async params => { + if (params.some(one => one.tx.id('hex') === ghost.tx.id('hex'))) { + scriptEntered.resolve() + await scriptGate.promise + } + return params.map(verifyParamsInJavaScript) + } + } + + const subject = coordinator(tracker, { concurrentTransactions: 2 }, verifier) + + // 1. Create a "ghost": a consumer cancels while GHOST's script check (a + // non-abortable backend call, per "does not free a non-abortable backend + // slot..." above) is still in flight. The concurrency slot it holds is + // not released until that call actually settles, well after the job + // itself has been finished and removed from `this.work`. + const ghostAbort = new AbortController() + const ghostPromise = subject.verify( + { beef: ghost.evidence, outputIndex: 0 }, + { signal: ghostAbort.signal } + ) + await scriptEntered.promise + ghostAbort.abort() + await expectCode(ghostPromise, 'cancelled') + expect(subject.getStats()).toMatchObject({ pendingTransactions: 0, activeAttempts: 1 }) + + // 2. Build an alternate (good-root) candidate for job A's txid up front. + const alternate = Transaction.fromBEEF(jobA.evidence) + const altSource = alternate.inputs[0].sourceTransaction + if (altSource === undefined) throw new Error('fixture source is missing') + altSource.merklePath = new MerklePath(height, [ + [ + { offset: 0, hash: altSource.id('hex'), txid: true }, + { offset: 1, hash: '5b'.repeat(32) } + ] + ]) + tracker.roots.add(altSource.merklePath.computeRoot(altSource.id('hex'))) + + // 3. Admit job A with its (invalid-root) first candidate. Together with + // the ghost, this uses both of the coordinator's concurrency slots. + const badReceipt = subject.verify({ beef: jobA.evidence, outputIndex: 0 }) + expect(subject.getStats().activeAttempts).toBe(2) + + // 4. A third, unrelated transaction arrives while the coordinator is at + // capacity, so it must wait for a free slot. + const displacer = subject.verify({ beef: jobD.evidence, outputIndex: 0 }) + + // 5. Job A's alternate (good) candidate is admitted onto the SAME, + // already-running job. + const goodReceipt = subject.verify({ beef: alternate.toBEEF(), outputIndex: 1 }) + + // Job A's first (bad-root) candidate fails and frees a slot; the waiting + // job D is admitted into it before job A's own retry can reclaim it. If + // that freed-and-immediately-reclaimed slot causes job A's own alternate + // candidate to be discarded instead of re-queued, both of job A's + // consumers are lost even though the alternate candidate is valid. + // (Promise.allSettled, rather than three sequential `await expect`s, + // ensures an early rejection here is observed as a failing assertion + // instead of an unhandled rejection while a sibling promise is pending.) + const [badOutcome, goodOutcome, displacerOutcome] = await Promise.allSettled([ + badReceipt, + goodReceipt, + displacer + ]) + expect(badOutcome).toMatchObject({ status: 'fulfilled', value: { outputIndex: 0 } }) + expect(goodOutcome).toMatchObject({ status: 'fulfilled', value: { outputIndex: 1 } }) + expect(displacerOutcome).toMatchObject({ status: 'fulfilled', value: { outputIndex: 0 } }) + + scriptGate.resolve() + }) + it('preflights approved duplicate and conflicting ancestry before warmed script work can be reused', async () => { const { tracker, ancestor, firstEvidence, secondEvidence } = await sharedAncestorFixture() const confirmed = ancestor.inputs[0].sourceTransaction From c9afc9d0cafcedc0d5b24fa02972ed7eb882d8c3 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 16:21:37 -0500 Subject: [PATCH 138/168] fix(wallet-toolbox): implement BHServiceClient.findChainTipHash Every TransactionEvidenceCoordinator attempt calls ChaintracksChainTracker.getVerificationContextToken(), which requires chaintracks.findChainTipHash(). BHServiceClient implemented that method as `throw new Error('Not implemented')` even though it already exposes a working findChainTipHeader() against /api/v1/chain/tip/longest. Any wallet using a BHServiceClient as options.chaintracks, or as a LocalChainTracker participating source, broke on every verification attempt. Fix: implement findChainTipHash() by delegating to the existing findChainTipHeader() and returning its hash, matching the same pattern GoChaintracksServiceClient already uses for the same interface method. No other ChaintracksClientApi contract surface changes. Added a regression test on BHServiceClient confirming findChainTipHash() resolves to findChainTipHeader()'s hash instead of throwing, and a regression test on ChaintracksChainTracker confirming getVerificationContextToken() succeeds end-to-end when backed by a BHServiceClient. Both fail with "Not implemented" on the old code and pass with the fix. Co-Authored-By: Claude Fable 5.1 --- .../src/services/chaintracker/BHServiceClient.ts | 2 +- .../chaintracker/__tests/BHServiceClient.test.ts | 13 +++++++++++-- .../__tests/ChaintracksChainTracker.test.ts | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts index 2c442f453..467643192 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/BHServiceClient.ts @@ -195,7 +195,7 @@ export class BHServiceClient implements ChaintracksServiceClient { } async findChainTipHash(): Promise { - throw new Error('Not implemented') + return (await this.findChainTipHeader()).hash } async subscribeHeaders(_listener: HeaderListener): Promise { diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/BHServiceClient.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/BHServiceClient.test.ts index c3d8e6722..d054acb2e 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/BHServiceClient.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/BHServiceClient.test.ts @@ -8,7 +8,7 @@ const queried = 'aa'.repeat(32) const canonical = 'bb'.repeat(32) const reorged = 'cc'.repeat(32) -function header(merkleRoot: string): BlockHeader { +function header(merkleRoot: string, hash: string = 'dd'.repeat(32)): BlockHeader { return { version: 1, previousHash: '00'.repeat(32), @@ -17,7 +17,7 @@ function header(merkleRoot: string): BlockHeader { bits: 1, nonce: 1, height, - hash: 'dd'.repeat(32) + hash } } @@ -89,6 +89,15 @@ describe('BHServiceClient height-root cache', () => { expect(client.cache[height]).toBeUndefined() }) + test('findChainTipHash returns the active chain tip header hash', async () => { + const client = bhsClient() + const tipHash = 'ee'.repeat(32) + jest.spyOn(client, 'findChainTipHeader').mockResolvedValue(header(canonical, tipHash)) + + await expect(client.findChainTipHash()).resolves.toBe(tipHash) + expect(client.findChainTipHeader).toHaveBeenCalledTimes(1) + }) + test('LocalChainTracker fallback re-reads BHServiceClient roots instead of the queried-root cache', async () => { const fallback = bhsClient() jest.spyOn(fallback, 'findHeaderForHeight').mockResolvedValue(header(canonical)) diff --git a/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts index 34ef1bbfa..df8598e38 100644 --- a/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts +++ b/packages/wallet/wallet-toolbox/src/services/chaintracker/__tests/ChaintracksChainTracker.test.ts @@ -1,5 +1,6 @@ import { ChaintracksChainTracker } from '../index.all' import { ChaintracksServiceClient } from '../chaintracks/ChaintracksServiceClient' +import { BHServiceClient } from '../BHServiceClient' import { sdk } from '../../../index.client' import { BlockHeader } from '../../../sdk/WalletServices.interfaces' @@ -216,6 +217,19 @@ describe('ChaintracksChaintracker tests', () => { expect(provider.subscribeReorgs).not.toHaveBeenCalled() }) + test('obtains a verification context token from a BHServiceClient chaintracks source', async () => { + const provider = new BHServiceClient('main', 'https://headers.example', 'test-key') + expect(provider.supportsReorgEvents).toBe(false) + const tipHash = 'aa'.repeat(32) + jest.spyOn(provider, 'findChainTipHeader').mockResolvedValue({ ...HEADER_877599, hash: tipHash }) + const subscribeReorgs = jest.spyOn(provider, 'subscribeReorgs') + const tracker = new ChaintracksChainTracker('main', provider) + + await expect(tracker.getVerificationContextToken()).resolves.toContain(tipHash) + expect(provider.findChainTipHeader).toHaveBeenCalledTimes(1) + expect(subscribeReorgs).not.toHaveBeenCalled() + }) + test('does not hide a registration failure from a built-in client that promises reorg events', async () => { class PromisingEventsClient extends ChaintracksServiceClient { override readonly supportsReorgEvents = true From 83c067098067973b813738abff9f51c3b689fb73 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 16:25:17 -0500 Subject: [PATCH 139/168] docs: changelog entries for F1/F2 review-finding fixes Document the TransactionEvidenceCoordinator candidate-requeue fix and the BHServiceClient.findChainTipHash implementation in their packages' CHANGELOG.md, matching this repo's convention of recording behavior- affecting fixes there alongside the commit history. Co-Authored-By: Claude Fable 5.1 --- packages/sdk/CHANGELOG.md | 7 +++++++ packages/wallet/wallet-toolbox/CHANGELOG.md | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 06fbeea69..e78e6e082 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -328,6 +328,13 @@ All notable changes to this project will be documented in this file. The format ### Fixed +- Re-queue a `TransactionEvidenceCoordinator` candidate that was displaced by a + concurrency-limited attempt instead of discarding it. A same-txid alternate + candidate already admitted to a job could previously be lost without ever + being tried when every concurrency slot was in use at the moment of its + retry, causing an otherwise-valid candidate to fail with `limit`. No public + API change; internal candidate/byte accounting is unaffected. + - Use asynchronous platform SHA-256 for ProtoWallet signature payloads of at least 64 KiB. Preserve deterministic signatures, direct digests, short input behavior, and portable fallback over a snapshot if native hashing is unavailable diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index 3cb2cba20..c6bf93540 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -6,6 +6,14 @@ attention to changes that materially alter behavior or extend functionality. ## wallet-toolbox (unreleased) +- Implement `BHServiceClient.findChainTipHash()` by delegating to its existing + `findChainTipHeader()` call against `/api/v1/chain/tip/longest`, instead of + throwing `Not implemented`. `ChaintracksChainTracker.getVerificationContextToken()` + calls `findChainTipHash()` on every verification attempt, so any wallet + configured with a `BHServiceClient` as `options.chaintracks`, or as a + `LocalChainTracker` participating source, previously failed on every + attempt. No `ChaintracksClientApi` contract change; no migration required. + - Preserve valid compound proofs with multiple marked transactions. Rotate unresolved proof repairs behind waiting heights across monitor restarts, and retain retries that become temporarily ineligible after the chain tip From 640661bd1d94bd082df7555b16e49bd008b3d20b Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 16:36:12 -0500 Subject: [PATCH 140/168] style: restore newline dropped by the restack conflict resolution Whitespace only; the merged release-notes content is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- governance/package-release-notes.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index ff4f23af7..0b54842ee 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -127,7 +127,8 @@ "publishedVersion": "2.2.1", "releaseType": "minor", "summary": "Adds an engine lookup-result cardinality ceiling before transaction/proof hydration and an optional TopicManager abort hook for provisional admission state when strict transaction broadcast fails. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Adds optional v1 admission/storage capability types, deterministic semantic identity and exact integer helpers, recovery fences and cursor-publication predicates, with shared reference conformance fixtures. Adds an opt-in MongoDB foundation for validated scoped schema bootstrap, content-addressed payload publication, transactionally guarded references, and recoverable payload collection. Adds bounded BASM peer JSON validation, canonical header/proof/raw binding, optional trusted full-block-count position checks, and classified capability/resource errors. Forward sync pages fit the standard 1,000-anchor HTTP limit. Claimed admitted-list indices are bound to the compound path even when every remote txid is already local, and inclusion uses chain-tracker root/height rather than coinbase maturity. Adds an opt-in Mongo AdmissionStorage adapter and Engine submit path that uses commitAdmission when overlay-admission-v1 is advertised, with enlisted indexes or a durable projection outbox and majority ACK.", - "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Knex/SQL submit remains the default path; Engine uses commitAdmission only when storage advertises a complete overlay-admission-v1 provider. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. BASM admission therefore submits in the historical-tx-no-spv mode, so topic managers that branch on the submission mode observe historical-tx-no-spv rather than historical-tx for BASM-admitted transactions. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included." }, + "migration": "Existing Engine and TopicManager implementations remain valid. Lookup results default to 1,000 formulas; pass -1 only when an equivalent deployment bound exists. Topic managers whose validation creates provisional external state should implement abortAdmissibleOutputs, while read-only managers require no change. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. No migration is required for the additive storage capability. Knex/SQL submit remains the default path; Engine uses commitAdmission only when storage advertises a complete overlay-admission-v1 provider. MongoDB is an optional peer: applications that import a Mongo storage deep entry point must install mongodb ^7.5.0 and explicitly operate the replica-set foundation. No default-adapter change, SQL-to-Mongo migration, or mixed-writer database topology is included. Existing Engine, TopicManager and header providers remain valid. BASM reconciliation requires a canonical header resolver; blockTransactionCount is optional and must come from the trusted provider for the same block hash. Header-only proof position assurance is reported explicitly. A remote admitted list is bound to its compound Merkle path before it is treated as divergence or repair evidence. Proof inclusion checks the canonical root at the claimed height and does not apply coinbase 100-block spendability; historical submit still runs standard SPV. BASM admission therefore submits in the historical-tx-no-spv mode, so topic managers that branch on the submission mode observe historical-tx-no-spv rather than historical-tx for BASM-admitted transactions. Untrusted bootstrap prefixes and over-limit proof/raw requests fail closed pending durable recovery work; automatic sync remains off and no schema migration is included." + }, { "name": "@bsv/overlay-discovery-services", "publishedVersion": "2.1.1", From a4259054cb43b0091716d5117f00493dc61d8ccf Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 17:00:17 -0500 Subject: [PATCH 141/168] fix(overlay): exclude dupe topics from Mongo admission commit (F1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildOverlayAdmissionPlan's `accepted` filter treated an already-applied (isDupe) topic the same as a genuinely new admission, so it still received an identity entry, a decision, and a place in the commit. Any retry that resubmits an already-applied topic under a different admission operation — OverlayGASPStorage.finalizeGraph submitting 'historical-tx-no-spv' after a live admit, a live resubmit of a historically synced tx, or a multi-topic retry that mixes an applied topic with a new one — produces a plan whose operationId differs from the one recorded when that topic was first applied. MongoAdmissionStorage.assertAppliedAvailable rejects any applied row whose admissionId belongs to a different operation, so the retry threw "Overlay admission rejected: invalid-plan" (or 'digest-mismatch') instead of being idempotent. Fix: exclude isDupe topics from the plan's identity/decisions via a new selectNewAdmissionTopics() helper, shared by EngineAdmission.ts and Engine.ts. When every remaining topic is a dupe or a failure, Engine.acknowledgeOverlayAdmission now returns the in-memory STEAK (already computed from validation) without building or committing a plan at all. On a mixed set, only the new topics are committed; the STEAK returned still reports each dupe topic as accepted-with-nothing-new, matching what the classic (non-admission) storage path already returns for a dupe. Updated one existing EngineAdmission unit test that had encoded the old (buggy) contract directly — asserting that an isDupe-only validation still produced a decision with merkle metadata — to use a real admission instead, since a dupe-only plan is no longer a supported call shape (documented by a new test asserting it throws). Tests: new Engine-level cases for all-dupe, mixed dupe/new, and resubmit-after-historical, plus a real MongoAdmissionStorage-backed regression (via Engine + MongoOverlayStorage) that reproduces the exact multi-topic "invalid-plan" rejection this fixes. New EngineAdmission unit tests cover buildOverlayAdmissionPlan's dupe exclusion directly. Full @bsv/overlay suite (mock + mongo/**) passes. Co-Authored-By: Claude Fable 5.1 --- packages/overlays/overlay/CHANGELOG.md | 12 +++ packages/overlays/overlay/src/Engine.ts | 14 +++ .../overlays/overlay/src/EngineAdmission.ts | 40 +++++-- .../src/__tests/Engine.admission.test.ts | 62 +++++++++++ .../src/__tests/EngineAdmission.test.ts | 45 +++++++- .../EngineAdmissionDupeRetry.mongo.test.ts | 102 ++++++++++++++++++ 6 files changed, 266 insertions(+), 9 deletions(-) create mode 100644 packages/overlays/overlay/src/__tests/mongo/EngineAdmissionDupeRetry.mongo.test.ts diff --git a/packages/overlays/overlay/CHANGELOG.md b/packages/overlays/overlay/CHANGELOG.md index 2804aae4b..0ccd7e84d 100644 --- a/packages/overlays/overlay/CHANGELOG.md +++ b/packages/overlays/overlay/CHANGELOG.md @@ -56,6 +56,18 @@ All notable changes to this project will be documented in this file. The format ### Fixed - (Document bugs that were fixed since the last release.) +- Exclude already-applied (dupe) topics from the Mongo admission plan's + identity, decisions and commit, instead of resubmitting them for admission. + A dupe topic no longer reaches `commitAdmission`; when every remaining + topic is a dupe or a failure, `Engine.submit` returns the in-memory STEAK + without building or committing a plan at all, and on a mixed submission + only the genuinely new topics are committed while the STEAK still reports + the dupe as accepted-with-nothing-new. Previously a resubmitted dupe was + still treated as accepted and given a decision, which + `MongoAdmissionStorage.assertAppliedAvailable` rejects whenever that topic + was already applied under a different admission operation — breaking + historical-then-live resubmits and multi-topic retries with + `Overlay admission rejected: invalid-plan` or `digest-mismatch`. ### Security - (Notify of any improvements related to security vulnerabilities or potential risks.) diff --git a/packages/overlays/overlay/src/Engine.ts b/packages/overlays/overlay/src/Engine.ts index 22091597d..fa3d0a6ef 100644 --- a/packages/overlays/overlay/src/Engine.ts +++ b/packages/overlays/overlay/src/Engine.ts @@ -49,6 +49,7 @@ import { buildOverlayAdmissionPlan, getOverlayAdmissionHost, overlayAdmissionMode, + selectNewAdmissionTopics, waitForAdmissionReceipt } from './EngineAdmission.js' @@ -1104,6 +1105,19 @@ export class Engine { if (onSteakReady !== undefined) onSteakReady(steak) return steak } + // A topic counts toward `anyTopicAccepted` when it is a dupe (so the + // caller still gets broadcast/propagation for a retried submission), but + // a dupe must never be resubmitted for admission: it was already + // committed under a (possibly different) operation, and re-including it + // here would make commitAdmission reject the whole plan. When nothing + // left over is a genuinely new admission, skip the plan/commit entirely + // and hand back the STEAK already computed from validation — it reports + // each dupe topic as accepted-with-nothing-new, same as the classic + // (non-admission) storage path does. + if (selectNewAdmissionTopics(validations, failedTopics).length === 0) { + if (onSteakReady !== undefined) onSteakReady(steak) + return steak + } const applied = await this.buildAppliedTransactionRecord(tx) const buildPlan = async () => await buildOverlayAdmissionPlan({ diff --git a/packages/overlays/overlay/src/EngineAdmission.ts b/packages/overlays/overlay/src/EngineAdmission.ts index 48f37c702..f4ffb15ca 100644 --- a/packages/overlays/overlay/src/EngineAdmission.ts +++ b/packages/overlays/overlay/src/EngineAdmission.ts @@ -112,6 +112,30 @@ async function localPayload( } } +/** + * Selects the topics that require a new admission decision: topics that did + * not fail validation, are not already-applied dupes, and actually admit or + * retain something (or consume a previously-admitted coin). Already-applied + * (dupe) topics are excluded so that a retry — including a historical GASP + * replay after a live admit, or a multi-topic resubmit where only some + * topics were previously applied — never re-submits an admission decision + * for a topic that MongoAdmissionStorage.assertAppliedAvailable would reject + * as belonging to a different operation. + */ +export function selectNewAdmissionTopics( + validations: readonly T[], + failedTopics: Set +): T[] { + return validations.filter( + validation => + !failedTopics.has(validation.topic) && + !validation.isDupe && + (validation.admissibleOutputs.outputsToAdmit.length > 0 || + validation.admissibleOutputs.coinsToRetain.length > 0 || + validation.previousCoins.length > 0) + ) +} + export async function buildOverlayAdmissionPlan(input: { host: OverlayAdmissionHost tx: Transaction @@ -132,14 +156,14 @@ export async function buildOverlayAdmissionPlan(input: { merkleRoot?: string } }): Promise { - const accepted = input.validations.filter( - validation => - !input.failedTopics.has(validation.topic) && - (validation.isDupe || - validation.admissibleOutputs.outputsToAdmit.length > 0 || - validation.admissibleOutputs.coinsToRetain.length > 0 || - validation.previousCoins.length > 0) - ) + // Only topics with a genuinely new admission get an identity/decision entry + // and take part in the commit. Dupe topics are still reflected in `steak` + // below (with their empty admissibleOutputs), matching what the classic + // storage path already returns for a dupe, but they must never be + // resubmitted to commitAdmission: MongoAdmissionStorage.assertAppliedAvailable + // rejects an applied row whose admissionId belongs to a different + // operation, which a dupe re-included here would trigger on retry. + const accepted = selectNewAdmissionTopics(input.validations, input.failedTopics) const identityTopics = accepted.map(validation => ({ topic: validation.topic, policyId: OVERLAY_ENGINE_POLICY_ID diff --git a/packages/overlays/overlay/src/__tests/Engine.admission.test.ts b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts index 0c6229082..9dad99fb5 100644 --- a/packages/overlays/overlay/src/__tests/Engine.admission.test.ts +++ b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts @@ -257,6 +257,68 @@ describe('Engine admission submit', () => { expect(onReady).toHaveBeenCalledWith(steak) }) + test('does not call commitAdmission and returns the in-memory STEAK when every topic is a dupe', async () => { + mockStorage.doesAppliedTransactionExist = jest.fn(async () => true) + const onReady = jest.fn() + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker + ) + const steak = await engine.submit({ beef: exampleBeef, topics: ['Hello'] }, onReady) + expect(steak).toEqual({ Hello: { outputsToAdmit: [], coinsToRetain: [] } }) + expect(commitAdmission).not.toHaveBeenCalled() + expect(onReady).toHaveBeenCalledWith(steak) + expect(mockTopicManager.identifyAdmissibleOutputs).not.toHaveBeenCalled() + }) + + test('commits only the new topic on a mixed dupe/new-admission submission, while STEAK still reports the dupe', async () => { + mockStorage.doesAppliedTransactionExist = jest.fn( + async (params: { topic: string }) => params.topic === 'Dupe' + ) + const engine = new Engine( + { Hello: mockTopicManager, Dupe: mockTopicManager }, + { Hello: mockLookupService, Dupe: mockLookupService }, + mockStorage, + mockChainTracker + ) + const steak = await engine.submit({ beef: exampleBeef, topics: ['Hello', 'Dupe'] }) + expect(commitAdmission).toHaveBeenCalledTimes(1) + const plan = commitAdmission.mock.calls[0][0] as AdmissionCommit + expect(plan.decisions.map(decision => decision.topic)).toEqual(['Hello']) + expect(plan.identity.topics.map(entry => entry.topic)).toEqual(['Hello']) + const planSteak = JSON.parse(plan.steak) as Record + expect(planSteak.Dupe).toEqual({ outputsToAdmit: [], coinsToRetain: [], coinsRemoved: [] }) + expect(steak.Hello.outputsToAdmit).toEqual([0]) + expect(steak.Dupe).toEqual({ outputsToAdmit: [], coinsToRetain: [], coinsRemoved: [] }) + }) + + test('resubmit after historical admission (same tx, same topic) does not re-commit and does not throw', async () => { + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker + ) + const first = await engine.submit( + { beef: exampleBeef, topics: ['Hello'] }, + undefined, + 'historical-tx-no-spv' + ) + expect(first.Hello.outputsToAdmit).toEqual([0]) + expect(commitAdmission).toHaveBeenCalledTimes(1) + + mockStorage.doesAppliedTransactionExist = jest.fn(async () => true) + const second = await engine.submit( + { beef: exampleBeef, topics: ['Hello'] }, + undefined, + 'current-tx' + ) + expect(commitAdmission).toHaveBeenCalledTimes(1) + expect(second).toEqual({ Hello: { outputsToAdmit: [], coinsToRetain: [] } }) + }) + test('uses host payload, fence, and enlisted index hooks', async () => { const publishAdmissionPayload = jest.fn( async (input: { kind: string; bytes: Uint8Array; txid?: string }) => ({ diff --git a/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts b/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts index 85614c302..7dd05a7ab 100644 --- a/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts +++ b/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts @@ -237,7 +237,7 @@ describe('Engine overlay admission helpers', () => { beef: proven.toBEEF(), topics: ['Hello'], mode: 'historical', - validations: [validation({ isDupe: true, outputsToAdmit: [] })], + validations: [validation()], failedTopics: new Set(), lookupServices, includePropagation: true @@ -248,6 +248,49 @@ describe('Engine overlay admission helpers', () => { expect(plan.decisions[0].applied.firstSeenHeight).toBeDefined() }) + test('excludes a dupe topic from identity and decisions but keeps it, empty, in steak', async () => { + const plan = await buildOverlayAdmissionPlan({ + host: host(), + tx: exampleTX, + txid: exampleTxid, + beef: exampleBeef, + topics: ['Hello', 'World'], + mode: 'live', + validations: [ + validation({ isDupe: true, outputsToAdmit: [] }), + validation({ topic: 'World' }) + ], + failedTopics: new Set(), + lookupServices, + includePropagation: false + }) + expect(plan.identity.topics.map(entry => entry.topic)).toEqual(['World']) + expect(plan.decisions.map(decision => decision.topic)).toEqual(['World']) + expect(JSON.parse(plan.steak).Hello).toEqual({ + outputsToAdmit: [], + coinsToRetain: [], + coinsRemoved: [] + }) + expect(JSON.parse(plan.steak).World.outputsToAdmit).toEqual([0]) + }) + + test('throws when every topic is a dupe or failed, since callers must not commit an empty plan', async () => { + await expect( + buildOverlayAdmissionPlan({ + host: host(), + tx: exampleTX, + txid: exampleTxid, + beef: exampleBeef, + topics: ['Hello'], + mode: 'live', + validations: [validation({ isDupe: true, outputsToAdmit: [] })], + failedTopics: new Set(), + lookupServices, + includePropagation: false + }) + ).rejects.toThrow('Overlay admission plan has no topics') + }) + test('marks unmatched previous coins stale and retains consumed coins', async () => { const stale = await buildOverlayAdmissionPlan({ host: host(), diff --git a/packages/overlays/overlay/src/__tests/mongo/EngineAdmissionDupeRetry.mongo.test.ts b/packages/overlays/overlay/src/__tests/mongo/EngineAdmissionDupeRetry.mongo.test.ts new file mode 100644 index 000000000..b3b28038b --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/EngineAdmissionDupeRetry.mongo.test.ts @@ -0,0 +1,102 @@ +import { Transaction, type AdmittanceInstructions } from '@bsv/sdk' +import { Engine } from '../../Engine.js' +import type { LookupService } from '../../LookupService.js' +import type { TopicManager } from '../../TopicManager.js' +import { MongoOverlayStorage } from '../../storage/mongo/MongoOverlayStorage.js' +import { bootstrapMongoOverlay } from '../../storage/mongo/MongoSchema.js' +import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' + +// Same fixture BEEF used throughout the admission test suite: a single, +// unconfirmed transaction whose one input spends a proven ancestor output. +const BRC62Hex = + '0100beef01fe636d0c0007021400fe507c0c7aa754cef1f7889d5fd395cf1f785dd7de98eed895dbedfe4e5bc70d1502ac4e164f5bc16746bb0868404292ac8318bbac3800e4aad13a014da427adce3e010b00bc4ff395efd11719b277694cface5aa50d085a0bb81f613f70313acd28cf4557010400574b2d9142b8d28b61d88e3b2c3f44d858411356b49a28a4643b6d1a6a092a5201030051a05fc84d531b5d250c23f4f886f6812f9fe3f402d61607f977b4ecd2701c19010000fd781529d58fc2523cf396a7f25440b409857e7e221766c57214b1d38c7b481f01010062f542f45ea3660f86c013ced80534cb5fd4c19d66c56e7e8c5d4bf2d40acc5e010100b121e91836fd7cd5102b654e9f72f3cf6fdbfd0b161c53a9c54b12c841126331020100000001cd4e4cac3c7b56920d1e7655e7e260d31f29d9a388d04910f1bbd72304a79029010000006b483045022100e75279a205a547c445719420aa3138bf14743e3f42618e5f86a19bde14bb95f7022064777d34776b05d816daf1699493fcdf2ef5a5ab1ad710d9c97bfb5b8f7cef3641210263e2dee22b1ddc5e11f6fab8bcd2378bdd19580d640501ea956ec0e786f93e76ffffffff013e660000000000001976a9146bfd5c7fbe21529d45803dbcf0c87dd3c71efbc288ac0000000001000100000001ac4e164f5bc16746bb0868404292ac8318bbac3800e4aad13a014da427adce3e000000006a47304402203a61a2e931612b4bda08d541cfb980885173b8dcf64a3471238ae7abcd368d6402204cbf24f04b9aa2256d8901f0ed97866603d2be8324c2bfb7a37bf8fc90edd5b441210263e2dee22b1ddc5e11f6fab8bcd2378bdd19580d640501ea956ec0e786f93e76ffffffff013c660000000000001976a9146bfd5c7fbe21529d45803dbcf0c87dd3c71efbc288ac0000000000' + +const mockChainTracker = { + isValidRootForHeight: async () => true, + currentHeight: async () => 800000 +} + +function admittingManager(): TopicManager { + return { + identifyAdmissibleOutputs: async (): Promise => ({ + outputsToAdmit: [0], + coinsToRetain: [] + }), + getDocumentation: async () => 'docs', + getMetaData: async () => ({ name: 'Mock', shortDescription: 'Mock' }) + } +} + +function noopLookup(): LookupService { + return { + outputAdmittedByTopic: async () => {}, + outputSpent: async () => {}, + lookup: async () => [], + outputNoLongerRetainedInHistory: async () => {}, + outputEvicted: async () => {}, + admissionMode: 'locking-script', + spendNotificationMode: 'none', + getDocumentation: async () => 'docs', + getMetaData: async () => ({ name: 'Mock', shortDescription: 'Mock' }) + } +} + +describe('Engine admission retries against real MongoAdmissionStorage', () => { + let fixture: MongoReplicaFixture + + beforeAll(async () => { + fixture = await createMongoReplicaFixture() + await bootstrapMongoOverlay(fixture.db, fixture.scope) + }, 120000) + + afterAll(async () => { + await fixture.close() + }, 60000) + + test( + 'a multi-topic resubmit that overlaps an already-applied topic commits only the new topic instead of rejecting invalid-plan', + async () => { + const storage = new MongoOverlayStorage(fixture.db, fixture.scope) + // getOverlayAdmissionHost() lifts these off the storage instance into a + // plain object, so they must be pre-bound here or they lose `this` + // when EngineAdmission invokes them later — unrelated to the F1 fix + // under test, this is only how MongoOverlayStorage's admission host is + // wired up for direct use outside of a DI container. + storage.publishAdmissionPayload = storage.publishAdmissionPayload.bind(storage) + storage.enlistedIndexTargets = storage.enlistedIndexTargets.bind(storage) + storage.getHistoryFence = storage.getHistoryFence.bind(storage) + const engine = new Engine( + { TopicA: admittingManager(), TopicC: admittingManager() }, + { TopicA: noopLookup(), TopicC: noopLookup() }, + storage, + mockChainTracker + ) + + const tx = Transaction.fromHexBEEF(BRC62Hex) + const beef = tx.toBEEF() + + // Live admit under a single topic, TopicA. + const first = await engine.submit( + { beef, topics: ['TopicA'] }, + undefined, + 'historical-tx-no-spv' + ) + expect(first.TopicA.outputsToAdmit).toEqual([0]) + + // A later resubmission of the same tx names TopicA again (now a dupe) + // alongside a brand-new topic, TopicC — the "multi-topic retry" shape + // the finding describes. This must not throw + // "Overlay admission rejected: invalid-plan" / "digest-mismatch". + const second = await engine.submit( + { beef, topics: ['TopicA', 'TopicC'] }, + undefined, + 'historical-tx-no-spv' + ) + expect(second.TopicC.outputsToAdmit).toEqual([0]) + expect(second.TopicA).toEqual({ outputsToAdmit: [], coinsToRetain: [], coinsRemoved: [] }) + + await storage.close() + }, + 30000 + ) +}) From 907739402fcbb9b36daa6c018f49ef49e690ab17 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 17:09:35 -0500 Subject: [PATCH 142/168] fix(overlay-mongo): never surface an evicted output as live (F2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MongoOverlayStorage.toOutput mapped `spent: document.state === 'spent'`, so an evicted document (state: 'evicted', kept for audit/history rather than deleted) would map to `spent: false` — a live, unspent output — if it ever reached this function. findOutput and findOutputsForTransaction already excluded `evicted` at the query level, but that meant the "evicted is never live" invariant lived only in two separate query filters rather than in the one function actually responsible for shaping an Output, so a future read path that forgot the filter would silently resurface deleted UTXOs. toOutput also hard-coded outputsConsumed: [] and consumedBy: [], so the consumption edges written during an admission commit (or via updateConsumedBy on the classic path) were invisible to every reader. Fix: toOutput now returns `Output | null`, returning null for any `state: 'evicted'` document; all three read paths (findOutput, findOutputsForTransaction, findUTXOsForTopic) go through a shared toOutputs() helper that drops null mappings the same way, so the invariant holds regardless of each query's own filter. toOutput now also hydrates outputsConsumed/consumedBy by querying the consumptionEdges collection persisted by MongoAdmissionStorage.insertEdge and MongoOverlayStorage.updateConsumedBy. Tests: a direct test of toOutput (the two current public read paths already filter evicted at the query level, so this is the only way to observe the mapping bug fail on current code) confirms an evicted document is never mapped to a live Output; a findOutput/ findOutputsForTransaction test confirms outputsConsumed/consumedBy are hydrated from persisted edges in both directions. Full @bsv/overlay mongo/** suite passes. Co-Authored-By: Claude Fable 5.1 --- packages/overlays/overlay/CHANGELOG.md | 12 ++ .../mongo/MongoOverlayStorageEviction.test.ts | 109 ++++++++++++++++++ .../src/storage/mongo/MongoOverlayStorage.ts | 85 ++++++++++++-- 3 files changed, 195 insertions(+), 11 deletions(-) create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageEviction.test.ts diff --git a/packages/overlays/overlay/CHANGELOG.md b/packages/overlays/overlay/CHANGELOG.md index 0ccd7e84d..4c2de3197 100644 --- a/packages/overlays/overlay/CHANGELOG.md +++ b/packages/overlays/overlay/CHANGELOG.md @@ -68,6 +68,18 @@ All notable changes to this project will be documented in this file. The format was already applied under a different admission operation — breaking historical-then-live resubmits and multi-topic retries with `Overlay admission rejected: invalid-plan` or `digest-mismatch`. +- `MongoOverlayStorage` no longer maps an evicted output document to a live + `Output`. `toOutput` now returns `null` for a `state: 'evicted'` document + (evictions persist their row for audit/history, they do not delete it), + and every read path (`findOutput`, `findOutputsForTransaction`, + `findUTXOsForTopic`) drops a `null` mapping the same way it already + excludes `evicted` at the query level, so a future read path that forgets + that query-level filter cannot resurface an evicted UTXO as unspent. + `findOutput`/`findOutputsForTransaction` also hydrate `outputsConsumed` + and `consumedBy` from the `consumptionEdges` persisted by the admission + commit (or by `updateConsumedBy` on the classic path) instead of always + returning them empty, so Engine history/delete paths can see consumption + edges written during admission. ### Security - (Notify of any improvements related to security vulnerabilities or potential risks.) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageEviction.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageEviction.test.ts new file mode 100644 index 000000000..5ed1c1f39 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageEviction.test.ts @@ -0,0 +1,109 @@ +import { randomUUID } from 'node:crypto' +import type { Output } from '../../Output.js' +import { MongoOverlayStorage } from '../../storage/mongo/MongoOverlayStorage.js' +import { bootstrapMongoOverlay, MongoCollectionNames } from '../../storage/mongo/MongoSchema.js' +import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' + +// Direct access to the private mapping function this finding names: it is +// the single place responsible for never surfacing a non-serving (evicted) +// document as a live output. Every current public read path (findOutput, +// findOutputsForTransaction, findUTXOsForTopic) already filters `evicted` +// out of its Mongo query before a document ever reaches this function, so +// exercising the defect end-to-end through those methods cannot fail on the +// pre-fix code — the mapping bug is only observable by feeding `toOutput` +// the raw (evicted) document directly, which is exactly what a future read +// path that forgets the query-level filter would do. +type ToOutputHost = { + toOutput: (document: Record, includeBEEF: boolean) => Promise +} + +function randomHex(bytes: number): string { + return Array.from({ length: bytes }, () => Math.floor(Math.random() * 256).toString(16).padStart(2, '0')).join('') +} + +describe('MongoOverlayStorage eviction and consumption-edge hydration', () => { + let fixture: MongoReplicaFixture + let storage: MongoOverlayStorage + + beforeAll(async () => { + fixture = await createMongoReplicaFixture() + await bootstrapMongoOverlay(fixture.db, fixture.scope) + storage = new MongoOverlayStorage(fixture.db, fixture.scope) + }, 120000) + + afterAll(async () => { + await fixture.close() + }, 60000) + + test('toOutput never maps an evicted document to a live output', async () => { + const txid = randomHex(32) + const topic = 'Hello' + const output: Output = { + txid, + outputIndex: 0, + outputScript: [1, 2, 3], + satoshis: 1000, + topic, + spent: false, + outputsConsumed: [], + consumedBy: [] + } + await storage.insertOutput(output) + await storage.deleteOutput(txid, 0, topic) + + const raw = await fixture.db + .collection(MongoCollectionNames.outputs) + .findOne({ txid, topic, network: fixture.scope.network, genesisHash: fixture.scope.genesisHash }) + expect(raw?.state).toBe('evicted') + if (raw === null) throw new Error('expected the evicted document to still exist for audit') + + const mapped = await (storage as unknown as ToOutputHost).toOutput(raw, false) + expect(mapped).toBeNull() + }) + + test('findOutput and findOutputsForTransaction hydrate outputsConsumed/consumedBy from persisted consumption edges', async () => { + const topic = `Edges-${randomUUID()}` + const sourceTxid = randomHex(32) + const consumerTxid = randomHex(32) + + const source: Output = { + txid: sourceTxid, + outputIndex: 0, + outputScript: [9], + satoshis: 500, + topic, + spent: false, + outputsConsumed: [], + consumedBy: [] + } + const consumer: Output = { + txid: consumerTxid, + outputIndex: 0, + outputScript: [7], + satoshis: 400, + topic, + spent: false, + outputsConsumed: [], + consumedBy: [] + } + await storage.insertOutput(source) + await storage.insertOutput(consumer) + + // The admission transaction that created `consumer` also spent `source`; + // this is how both MongoAdmissionStorage.insertEdge (during commit) and + // MongoOverlayStorage.updateConsumedBy (classic path) persist that edge. + await storage.updateConsumedBy(sourceTxid, 0, topic, [{ txid: consumerTxid, outputIndex: 0 }]) + + const foundSource = await storage.findOutput(sourceTxid, 0, topic) + expect(foundSource?.outputsConsumed).toEqual([]) + expect(foundSource?.consumedBy).toEqual([{ txid: consumerTxid, outputIndex: 0 }]) + + const foundConsumer = await storage.findOutput(consumerTxid, 0, topic) + expect(foundConsumer?.outputsConsumed).toEqual([{ txid: sourceTxid, outputIndex: 0 }]) + expect(foundConsumer?.consumedBy).toEqual([]) + + const forTransaction = await storage.findOutputsForTransaction(consumerTxid) + expect(forTransaction).toHaveLength(1) + expect(forTransaction[0].outputsConsumed).toEqual([{ txid: sourceTxid, outputIndex: 0 }]) + }) +}) diff --git a/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts index 19669d5b8..7c2e1cdbb 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoOverlayStorage.ts @@ -208,9 +208,7 @@ export class MongoOverlayStorage implements Storage { state: { $ne: 'evicted' } }) .toArray() - return await Promise.all( - documents.map(async document => await this.toOutput(document, includeBEEF)) - ) + return await this.toOutputs(documents, includeBEEF) } async findUTXOsForTopic( @@ -232,9 +230,23 @@ export class MongoOverlayStorage implements Storage { .find(filter) .sort({ score: 1, _id: 1 }) if (limit !== undefined && limit > 0) query = query.limit(limit) - return await Promise.all( - (await query.toArray()).map(async document => await this.toOutput(document, includeBEEF)) + return await this.toOutputs(await query.toArray(), includeBEEF) + } + + /** + * Maps a batch of raw output documents to `Output`s, dropping any that + * `toOutput` treats as non-serving (see its docstring). Every current + * caller's query already excludes `state: 'evicted'`, so this filter is + * defense in depth rather than the primary guard. + */ + private async toOutputs( + documents: Array>, + includeBEEF: boolean + ): Promise { + const mapped = await Promise.all( + documents.map(async document => await this.toOutput(document, includeBEEF)) ) + return mapped.filter((output): output is Output => output !== null) } async deleteOutput(txid: string, outputIndex: number, topic: string): Promise { @@ -397,16 +409,35 @@ export class MongoOverlayStorage implements Storage { return Number(parsed) } - private async toOutput(document: Record, includeBEEF: boolean): Promise { + /** + * Maps a raw output document to an `Output`, or `null` when the document + * is not currently serving as overlay state. An admission eviction leaves + * its row in place (state: 'evicted') for audit/history, and it must never + * be reported as a live, unspent output — every caller that reaches here + * must treat a `null` result the same as "not found", regardless of + * whether its own query already excluded `evicted` rows. + */ + private async toOutput( + document: Record, + includeBEEF: boolean + ): Promise { + if (document.state === 'evicted') return null + const topic = document.topic as string + const txid = document.txid as string + const outputIndex = String(document.outputIndex) + const [outputsConsumed, consumedBy] = await Promise.all([ + this.readConsumptionEdges(topic, { consumerTxid: txid, consumerOutputIndex: outputIndex }, 'source'), + this.readConsumptionEdges(topic, { sourceTxid: txid, sourceOutputIndex: outputIndex }, 'consumer') + ]) const output: Output = { - txid: document.txid as string, - outputIndex: parseStorageOutputIndex(String(document.outputIndex)), + txid, + outputIndex: parseStorageOutputIndex(outputIndex), outputScript: await this.readScript(document), satoshis: this.toSafeNumber(decodeMongoUint64(document.satoshis as string), 'satoshis'), - topic: document.topic as string, + topic, spent: document.state === 'spent', - outputsConsumed: [], - consumedBy: [], + outputsConsumed, + consumedBy, score: this.toSafeNumber(decodeMongoUint64(document.score as string), 'score') } if (includeBEEF) { @@ -416,6 +447,38 @@ export class MongoOverlayStorage implements Storage { return output } + /** + * Reads consumption edges persisted for this output (by either + * MongoAdmissionStorage.insertEdge during a commit, or + * MongoOverlayStorage.updateConsumedBy on the classic storage path) and + * returns the opposite endpoint of each matching edge: querying by the + * consumer side returns `outputsConsumed` (what this output's transaction + * spent), and querying by the source side returns `consumedBy` (what has + * since spent this output). + */ + private async readConsumptionEdges( + topic: string, + match: { sourceTxid: string; sourceOutputIndex: string } | { consumerTxid: string; consumerOutputIndex: string }, + side: 'source' | 'consumer' + ): Promise> { + const edges = await this.db + .collection(MongoCollectionNames.consumptionEdges) + .find({ + network: this.admissionScope.network, + genesisHash: this.admissionScope.genesisHash, + nodeId: this.admissionScope.nodeId, + topic, + ...match + }) + .toArray() + return edges.map(edge => ({ + txid: (side === 'source' ? edge.sourceTxid : edge.consumerTxid) as string, + outputIndex: parseStorageOutputIndex( + String(side === 'source' ? edge.sourceOutputIndex : edge.consumerOutputIndex) + ) + })) + } + private async readScript(document: Record): Promise { const payloadId = document.scriptPayloadId if (typeof payloadId !== 'string') return [] From c2fa8a7c7cbf16c158ba3404e4dce7fd12702d1c Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 18:15:21 -0500 Subject: [PATCH 143/168] docs(sdk): state why a cached evidence recheck cannot be displaced The concurrency-limit signal is handled only in the candidate loop. The cached-positive recheck does not need the same handling: run() is reached solely from pump(), which checks the attempt limit synchronously before starting the job, and no await separates that check from the recheck attempt. Record the invariant next to the call so a later refactor of pump() or run() revisits it. Co-Authored-By: Claude Fable 5.1 --- packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts b/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts index fb7a520ce..6fec5c47e 100644 --- a/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts +++ b/packages/sdk/src/transaction/TransactionEvidenceCoordinator.ts @@ -292,6 +292,9 @@ export class TransactionEvidenceCoordinator { const cached = this.positives.get(job.txid) if (cached !== undefined) { try { + // run() is entered only from pump(), which has just confirmed a free + // attempt slot, and nothing awaits before this call. This first attempt + // therefore always starts; only later candidate attempts can be displaced. await this.attempt(job, async signal => await this.recheck(job, cached, signal)) this.check(job, job.controller.signal) this.finish(job, cached) From 2189b54d6f45586ec268f2cdc7d7b78375fb1e9a Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 18:15:48 -0500 Subject: [PATCH 144/168] chore(wallet): set platform budgets from measured evidence-branch bundles The ceilings carried on this branch were authored before main grew (#534, #547), so the merged wallet bundles exceeded them and CI stopped at the first over-budget dimension. Measured from the exact packed dependency graph at 83c067098 (JS bundle sizes verified byte-identical between macOS and hosted Linux; Hermes gzip estimated at the +1.3% Linux ratio recorded in #547): client Vite 1,759,717 raw / 416,284 gzip / 324,287 brotli client esbuild 1,372,320 raw / 378,741 gzip / 302,937 brotli mobile Metro 1,811,581 raw / 461,355 gzip / 355,871 brotli mobile Hermes 3,688,871 raw / 1,502,972 gzip / 1,166,821 brotli Only the dimensions that exceeded are raised, with about 0.25% headroom (1% on the Hermes gzip estimate, 0.5% on Hermes brotli run variance). Co-Authored-By: Claude Fable 5.1 --- .../wallet/wallet-toolbox/client/platform-budget.json | 4 ++-- .../wallet/wallet-toolbox/mobile/platform-budget.json | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index bfed7f28a..9c8f6ca16 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,12 +2,12 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1750000, + "raw": 1765000, "gzip": 430000, "brotli": 330000 }, "esbuild": { - "raw": 1360000, + "raw": 1376000, "gzip": 380000, "brotli": 320000 } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 174aa189b..388fe5344 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -2,14 +2,14 @@ "profile": "mobile", "maximumBytes": { "metro": { - "raw": 1810000, - "gzip": 455000, + "raw": 1817000, + "gzip": 463000, "brotli": 360000 }, "hermes": { "raw": 3750000, - "gzip": 1500000, - "brotli": 1160000 + "gzip": 1538000, + "brotli": 1173000 } } } From 43fc0b7aac926870a745190b0cd2243b667de789 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 18:16:02 -0500 Subject: [PATCH 145/168] chore(budgets): set ceilings from measured lookup discovery bundles Bounded discovery, the host queue and the streaming response reader grow every bundle that embeds @bsv/sdk. Measured from the exact packed dependency graph at the top of the ordered overlay wave (max of 5eef0fc67 and 907739402; JS bundle sizes verified byte-identical between macOS and hosted Linux; Hermes gzip estimated at the +1.3% Linux ratio from #547): @bsv/sdk UMD 591,705 raw / 168,531 gzip / 140,485 brotli @bsv/simple Vite 752,122 raw / 182,787 gzip / 150,699 brotli wallet client Vite 1,782,018 raw / 421,917 gzip / 328,724 brotli wallet client esbuild 1,389,928 raw / 383,620 gzip / 306,783 brotli wallet mobile Metro 1,834,234 raw / 466,804 gzip / 360,591 brotli wallet mobile Hermes 3,725,120 raw / 1,520,644 gzip / 1,181,384 brotli Only the dimensions that exceeded are raised, with about 0.25% headroom (1% on the Hermes gzip estimate, 0.5% on Hermes brotli run variance). Later branches in the wave move these bundles by under 150 bytes, so they inherit these ceilings unchanged. Co-Authored-By: Claude Fable 5.1 --- packages/helpers/simple/browser-budget.json | 4 ++-- packages/sdk/browser-budget.json | 2 +- .../wallet/wallet-toolbox/client/platform-budget.json | 6 +++--- .../wallet/wallet-toolbox/mobile/platform-budget.json | 10 +++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/helpers/simple/browser-budget.json b/packages/helpers/simple/browser-budget.json index 249bc9ee9..3e44031a0 100644 --- a/packages/helpers/simple/browser-budget.json +++ b/packages/helpers/simple/browser-budget.json @@ -25,9 +25,9 @@ ], "maximumBytes": { "vite": { - "raw": 750000, + "raw": 755000, "gzip": 185000, - "brotli": 150000 + "brotli": 151500 }, "esbuild": { "raw": 585000, diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index f72f855e8..822515491 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -30,7 +30,7 @@ "path": "dist/umd/bundle.js", "global": "bsv", "maximumBytes": { - "raw": 590000, + "raw": 594000, "gzip": 172000, "brotli": 145000 } diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 9c8f6ca16..b8d5b2bed 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,13 +2,13 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1765000, + "raw": 1787000, "gzip": 430000, "brotli": 330000 }, "esbuild": { - "raw": 1376000, - "gzip": 380000, + "raw": 1394000, + "gzip": 385000, "brotli": 320000 } } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 388fe5344..4ad3142a4 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -2,14 +2,14 @@ "profile": "mobile", "maximumBytes": { "metro": { - "raw": 1817000, - "gzip": 463000, - "brotli": 360000 + "raw": 1839000, + "gzip": 468000, + "brotli": 361500 }, "hermes": { "raw": 3750000, - "gzip": 1538000, - "brotli": 1173000 + "gzip": 1556000, + "brotli": 1188000 } } } From afa633e15689a188be88aae94d0e4721357bc1e4 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 18:16:14 -0500 Subject: [PATCH 146/168] test(overlay): name the historical-tx SPV test for what it covers GASP finalization already submits with historical-tx-no-spv; its historical-tx calls are dry-run admissibility checks. The restored guard protects public Engine.submit callers, so say that in the test title. Co-Authored-By: Claude Fable 5.1 --- .../overlays/overlay/src/__tests/BASMReconciliation.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts index 358fc8b88..be4c82b0e 100644 --- a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts +++ b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts @@ -495,7 +495,7 @@ describe('BASM reconciliation evidence binding', () => { ) }) - it('still runs SPV verification for a GASP-style historical-tx submission', async () => { + it('still runs SPV verification for a public historical-tx submission', async () => { const f = fixture() const proven = f.transactions[1] // A proof whose root is not the canonical root for this height: the chain From 2c9c99120be1d55c9a77f24cda9e5c6fac4477c7 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 19:01:32 -0500 Subject: [PATCH 147/168] test(overlay): exclude src/__tests support modules from coverage The repository's patch-coverage gate (scripts/patch-coverage.mjs, TEST_PATH pattern /(?:^|\/)__tests(?:__)?(?:\/|$)/) treats both `__tests/` and `__tests__/` directories as test code. This package's jest collectCoverageFrom only excluded `__tests__/**`, so the non-`.test.ts` support modules this PR adds under src/__tests/ (admission/ReferenceAdmissionStorage.ts, admission/AdmissionStorageContract.ts, mongo/MongoCommitResponseProxy.ts, mongo/MongoReplicaFixture.ts, ...) were instrumented and reported to Codecov as production code, understating the patch-coverage denominator correction the gate already applies elsewhere. Add '!src/**/__tests/**' so this package's own coverage collection matches the gate's definition of test code. Co-Authored-By: Claude Fable 5.1 --- packages/overlays/overlay/jest.config.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/overlays/overlay/jest.config.js b/packages/overlays/overlay/jest.config.js index 74140957e..8081537d4 100644 --- a/packages/overlays/overlay/jest.config.js +++ b/packages/overlays/overlay/jest.config.js @@ -14,5 +14,10 @@ export default { } ] }, - collectCoverageFrom: ['src/**/*.ts', '!src/**/*.test.ts', '!src/**/__tests__/**'] + collectCoverageFrom: [ + 'src/**/*.ts', + '!src/**/*.test.ts', + '!src/**/__tests__/**', + '!src/**/__tests/**' + ] } From ad041517aa0359caa5569711d71c36e4b52476b9 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 19:08:40 -0500 Subject: [PATCH 148/168] test(overlay): share the BASM reconciliation peer fixture Move the fake-peer fixture out of BASMReconciliation.test.ts so further fail-closed suites can drive the same engine, storage, chain tracker and fetch mock without duplicating them. No test behaviour changes. Co-Authored-By: Claude Fable 5.1 --- .../src/__tests/BASMReconciliation.test.ts | 111 +---------------- .../src/__tests/BASMReconciliationFixture.ts | 116 ++++++++++++++++++ 2 files changed, 117 insertions(+), 110 deletions(-) create mode 100644 packages/overlays/overlay/src/__tests/BASMReconciliationFixture.ts diff --git a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts index be4c82b0e..9cb71b7f9 100644 --- a/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts +++ b/packages/overlays/overlay/src/__tests/BASMReconciliation.test.ts @@ -1,116 +1,7 @@ -import { createHash } from 'node:crypto' import { LockingScript, MerklePath, Transaction } from '@bsv/sdk' -import { Engine } from '../Engine' import { BASM_ZERO_HASH, computeBasmRoot, computeTac } from '../BASM' import type { TopicBlockAnchor } from '../BASM' -import type { Storage } from '../storage/Storage' - -const topic = 'tm_basm_test' -const blockHeight = 100 -const blockHash = 'ab'.repeat(32) -const hashPair = (left: string, right: string): string => { - const bytes = Buffer.concat([ - Buffer.from(left, 'hex').reverse(), - Buffer.from(right, 'hex').reverse() - ]) - const first = createHash('sha256').update(bytes).digest() - return createHash('sha256').update(first).digest().reverse().toString('hex') -} - -function fixture() { - const transactions = [1, 2, 3, 4].map( - satoshis => - new Transaction(1, [], [{ satoshis, lockingScript: LockingScript.fromASM('OP_TRUE') }], 0) - ) - const ids = transactions.map(tx => tx.id('hex')) - const root = hashPair(hashPair(ids[0], ids[1]), hashPair(ids[2], ids[3])) - const path = new MerklePath(blockHeight, [ - ids.map((hash, offset) => ({ offset, hash, txid: true })), - [] - ]) - const admitted = [ - { txid: ids[1], blockIndex: 1 }, - { txid: ids[3], blockIndex: 3 } - ] - const anchor: TopicBlockAnchor = { - topic, - blockHeight, - blockHash, - basmRoot: computeBasmRoot(admitted), - admittedCount: 2, - tac: '' - } - anchor.tac = computeTac(BASM_ZERO_HASH, blockHash, anchor.basmRoot) - const responses: Record = { - '/requestTopicAnchorTip': anchor, - '/requestTopicAnchorRange': { topic, anchors: [anchor] }, - '/requestAdmittedList': { topic, blockHeight, blockHash, admitted }, - '/requestCompoundMerklePath': { - topic, - blockHeight, - txids: admitted.map(item => item.txid), - merklePath: path.toHex() - }, - '/requestRawTransactions': { - // Deliberately reversed: admission must still follow original block order. - transactions: [3, 1].map(index => ({ txid: ids[index], rawTx: transactions[index].toHex() })), - missing: [] - } - } - const storage = { - findTopicAnchorTip: jest.fn(async () => undefined), - findTopicBlockAnchor: jest.fn(async () => undefined), - findAdmittedTransactionsForBlock: jest.fn(async () => []) - } - const tracker = { - currentHeight: jest.fn(async () => 250), - isValidRootForHeight: jest.fn( - async (candidate: string, height: number) => candidate === root && height === blockHeight - ) - } - const engine = new Engine({}, {}, storage as unknown as Storage, tracker) - engine.syncConfiguration = { [topic]: ['https://peer.example'] } - engine.topicAnchorHeaderResolver = jest.fn(async height => ({ - blockHeight: height, - blockHash, - merkleRoot: root - })) - engine.logger = { ...console, error: jest.fn() } - const submit = jest.spyOn(engine, 'submit').mockResolvedValue({}) - const requests: Array<{ path: string; body: Record }> = [] - if (typeof globalThis.fetch !== 'function') { - Object.defineProperty(globalThis, 'fetch', { - configurable: true, - writable: true, - value: async () => { - throw new Error('unexpected network request') - } - }) - } - const fetchMock = jest.spyOn(globalThis, 'fetch').mockImplementation(async (url, init) => { - const endpoint = new URL(String(url)).pathname - requests.push({ - path: endpoint, - body: JSON.parse(String(init?.body)) as Record - }) - return new Response(JSON.stringify(responses[endpoint]), { status: 200 }) - }) - return { - engine, - responses, - anchor, - admitted, - transactions, - ids, - path, - root, - submit, - storage, - tracker, - requests, - fetchMock - } -} +import { blockHash, blockHeight, fixture, hashPair, topic } from './BASMReconciliationFixture' afterEach(() => jest.restoreAllMocks()) diff --git a/packages/overlays/overlay/src/__tests/BASMReconciliationFixture.ts b/packages/overlays/overlay/src/__tests/BASMReconciliationFixture.ts new file mode 100644 index 000000000..8cf7ad3d3 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/BASMReconciliationFixture.ts @@ -0,0 +1,116 @@ +import { createHash } from 'node:crypto' +import { LockingScript, MerklePath, Transaction } from '@bsv/sdk' +import { Engine } from '../Engine' +import { BASM_ZERO_HASH, computeBasmRoot, computeTac } from '../BASM' +import type { TopicBlockAnchor } from '../BASM' +import type { Storage } from '../storage/Storage' + +export const topic = 'tm_basm_test' +export const blockHeight = 100 +export const blockHash = 'ab'.repeat(32) + +export const hashPair = (left: string, right: string): string => { + const bytes = Buffer.concat([ + Buffer.from(left, 'hex').reverse(), + Buffer.from(right, 'hex').reverse() + ]) + const first = createHash('sha256').update(bytes).digest() + return createHash('sha256').update(first).digest().reverse().toString('hex') +} + +export function fixture() { + const transactions = [1, 2, 3, 4].map( + satoshis => + new Transaction(1, [], [{ satoshis, lockingScript: LockingScript.fromASM('OP_TRUE') }], 0) + ) + const ids = transactions.map(tx => tx.id('hex')) + const root = hashPair(hashPair(ids[0], ids[1]), hashPair(ids[2], ids[3])) + const path = new MerklePath(blockHeight, [ + ids.map((hash, offset) => ({ offset, hash, txid: true })), + [] + ]) + const admitted = [ + { txid: ids[1], blockIndex: 1 }, + { txid: ids[3], blockIndex: 3 } + ] + const anchor: TopicBlockAnchor = { + topic, + blockHeight, + blockHash, + basmRoot: computeBasmRoot(admitted), + admittedCount: 2, + tac: '' + } + anchor.tac = computeTac(BASM_ZERO_HASH, blockHash, anchor.basmRoot) + const responses: Record = { + '/requestTopicAnchorTip': anchor, + '/requestTopicAnchorRange': { topic, anchors: [anchor] }, + '/requestAdmittedList': { topic, blockHeight, blockHash, admitted }, + '/requestCompoundMerklePath': { + topic, + blockHeight, + txids: admitted.map(item => item.txid), + merklePath: path.toHex() + }, + '/requestRawTransactions': { + // Deliberately reversed: admission must still follow original block order. + transactions: [3, 1].map(index => ({ txid: ids[index], rawTx: transactions[index].toHex() })), + missing: [] + } + } + const storage = { + findTopicAnchorTip: jest.fn(async () => undefined), + findTopicBlockAnchor: jest.fn(async () => undefined), + findAdmittedTransactionsForBlock: jest.fn(async () => []) + } + const tracker = { + currentHeight: jest.fn(async () => 250), + isValidRootForHeight: jest.fn( + async (candidate: string, height: number) => candidate === root && height === blockHeight + ) + } + const engine = new Engine({}, {}, storage as unknown as Storage, tracker) + engine.syncConfiguration = { [topic]: ['https://peer.example'] } + engine.topicAnchorHeaderResolver = jest.fn(async height => ({ + blockHeight: height, + blockHash, + merkleRoot: root + })) + engine.logger = { ...console, error: jest.fn() } + const submit = jest.spyOn(engine, 'submit').mockResolvedValue({}) + const requests: Array<{ path: string; body: Record }> = [] + if (typeof globalThis.fetch !== 'function') { + Object.defineProperty(globalThis, 'fetch', { + configurable: true, + writable: true, + value: async () => { + throw new Error('unexpected network request') + } + }) + } + const fetchMock = jest.spyOn(globalThis, 'fetch').mockImplementation(async (url, init) => { + const endpoint = new URL(String(url)).pathname + requests.push({ + path: endpoint, + body: JSON.parse(String(init?.body)) as Record + }) + return new Response(JSON.stringify(responses[endpoint]), { status: 200 }) + }) + return { + engine, + responses, + anchor, + admitted, + transactions, + ids, + path, + root, + submit, + storage, + tracker, + requests, + fetchMock + } +} + +export type BASMFixture = ReturnType From 39b554f68b363e3f6d4ba01b9cd6698fe65eb95e Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 19:09:16 -0500 Subject: [PATCH 149/168] test(overlay): pin BASM_UNSUPPORTED provider guards Each BASM read endpoint fails closed with a TypeError carrying code 'BASM_UNSUPPORTED' when the storage backend does not implement the optional method behind it, and the capability check runs before the argument-shape guard on compound Merkle paths. Co-Authored-By: Claude Fable 5.1 --- .../src/__tests/BASMProviderGuards.test.ts | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 packages/overlays/overlay/src/__tests/BASMProviderGuards.test.ts diff --git a/packages/overlays/overlay/src/__tests/BASMProviderGuards.test.ts b/packages/overlays/overlay/src/__tests/BASMProviderGuards.test.ts new file mode 100644 index 000000000..bcf89dd42 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/BASMProviderGuards.test.ts @@ -0,0 +1,80 @@ +import { Engine } from '../Engine' +import type { Storage } from '../storage/Storage' + +const topic = 'tm_basm_guards' +const txid = 'ab'.repeat(32) + +/** A storage backend that implements none of the optional BASM read methods. */ +const engineWithout = (methods: Record = {}): Engine => + new Engine({}, {}, methods as unknown as Storage, 'scripts only') + +const expectUnsupported = async ( + operation: Promise, + message: string +): Promise => { + await expect(operation).rejects.toThrow(TypeError) + await expect(operation).rejects.toThrow(message) + await expect(operation).rejects.toMatchObject({ code: 'BASM_UNSUPPORTED' }) +} + +describe('BASM provider capability guards', () => { + it('refuses to serve a topic anchor tip without findTopicAnchorTip', async () => { + await expectUnsupported( + engineWithout().provideTopicAnchorTip(topic), + 'Storage does not support BASM topic anchor tips' + ) + }) + + it('refuses to serve a topic anchor range without findTopicBlockAnchors', async () => { + await expectUnsupported( + engineWithout().provideTopicAnchorRange(topic, 0, 0), + 'Storage does not support BASM topic anchor ranges' + ) + }) + + it('refuses to serve an admitted list without findAdmittedTransactionsForBlock', async () => { + await expectUnsupported( + engineWithout().provideAdmittedList(topic, 100), + 'Storage does not support BASM admitted lists' + ) + }) + + it('refuses to serve a compound Merkle path without findTransactionMerklePaths', async () => { + await expectUnsupported( + engineWithout().provideCompoundMerklePath(topic, 100, [txid]), + 'Storage does not support direct Merkle path lookup' + ) + }) + + it('refuses to serve raw transactions without findRawTransactions', async () => { + await expectUnsupported( + engineWithout().provideRawTransactions([txid]), + 'Storage does not support raw transaction lookup' + ) + }) + + it('checks capability before the empty-txid guard on compound Merkle paths', async () => { + // Order matters: an unsupported backend must report BASM_UNSUPPORTED rather + // than the argument-shape error a caller could mistake for a bad request. + await expectUnsupported( + engineWithout().provideCompoundMerklePath(topic, 100, []), + 'Storage does not support direct Merkle path lookup' + ) + }) + + it('still serves a tip and a range once the backend supports them', async () => { + const engine = engineWithout({ + findTopicAnchorTip: jest.fn(async () => undefined), + findTopicBlockAnchors: jest.fn(async () => []) + }) + await expect(engine.provideTopicAnchorTip(topic)).resolves.toEqual({ + topic, + blockHeight: -1, + tac: '00'.repeat(32) + }) + await expect(engine.provideTopicAnchorRange(topic, 5, 7)).resolves.toEqual({ + topic, + anchors: [] + }) + }) +}) From 3a6de8a8464adbb6c6b55b6d25a8669439dc8355 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 19:11:20 -0500 Subject: [PATCH 150/168] test(overlay): cover BASM tip-comparison and error reporting paths Exercises reconcileBASMWithPeer's not-ahead branch end to end: two empty topics match without a range request, an equal matching tip is only accepted after its local anchor is re-verified against the canonical header, a higher local tip reports divergence, and a tip whose anchor is missing, unsupported, TAC-mismatched or non-canonical fails closed. Also pins the report surface: "matched" once the page catches the local tip up, a stringified non-Error failure with no invented errorCode, and BASM_UNSUPPORTED surfacing through startBASMSync. Co-Authored-By: Claude Fable 5.1 --- .../src/__tests/BASMReconciliationFixture.ts | 8 +- .../src/__tests/BASMTipReconciliation.test.ts | 141 ++++++++++++++++++ 2 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 packages/overlays/overlay/src/__tests/BASMTipReconciliation.test.ts diff --git a/packages/overlays/overlay/src/__tests/BASMReconciliationFixture.ts b/packages/overlays/overlay/src/__tests/BASMReconciliationFixture.ts index 8cf7ad3d3..556d777f5 100644 --- a/packages/overlays/overlay/src/__tests/BASMReconciliationFixture.ts +++ b/packages/overlays/overlay/src/__tests/BASMReconciliationFixture.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto' import { LockingScript, MerklePath, Transaction } from '@bsv/sdk' import { Engine } from '../Engine' import { BASM_ZERO_HASH, computeBasmRoot, computeTac } from '../BASM' -import type { TopicBlockAnchor } from '../BASM' +import type { AdmittedTxRef, TopicAnchorTip, TopicBlockAnchor } from '../BASM' import type { Storage } from '../storage/Storage' export const topic = 'tm_basm_test' @@ -59,9 +59,9 @@ export function fixture() { } } const storage = { - findTopicAnchorTip: jest.fn(async () => undefined), - findTopicBlockAnchor: jest.fn(async () => undefined), - findAdmittedTransactionsForBlock: jest.fn(async () => []) + findTopicAnchorTip: jest.fn(async (): Promise => undefined), + findTopicBlockAnchor: jest.fn(async (): Promise => undefined), + findAdmittedTransactionsForBlock: jest.fn(async (): Promise => []) } const tracker = { currentHeight: jest.fn(async () => 250), diff --git a/packages/overlays/overlay/src/__tests/BASMTipReconciliation.test.ts b/packages/overlays/overlay/src/__tests/BASMTipReconciliation.test.ts new file mode 100644 index 000000000..27bfd4f8d --- /dev/null +++ b/packages/overlays/overlay/src/__tests/BASMTipReconciliation.test.ts @@ -0,0 +1,141 @@ +import { BASM_ZERO_HASH } from '../BASM' +import { blockHash, blockHeight, fixture, topic } from './BASMReconciliationFixture' + +afterEach(() => jest.restoreAllMocks()) + +describe('BASM reconciliation when the peer is not ahead', () => { + it('matches two empty topics without asking the peer for anchors', async () => { + const f = fixture() + f.responses['/requestTopicAnchorTip'] = { topic, blockHeight: -1, tac: BASM_ZERO_HASH } + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('matched') + expect(report.message).toBe('Topic anchor tips match') + expect(report.localTip).toEqual({ topic, blockHeight: -1, tac: BASM_ZERO_HASH }) + expect(report.remoteTip).toEqual({ topic, blockHeight: -1, tac: BASM_ZERO_HASH }) + // An empty remote tip has no anchor to cross-check, so the range endpoint + // must not be called and no local anchor lookup may happen either. + expect(f.requests.map(request => request.path)).toEqual(['/requestTopicAnchorTip']) + expect(f.storage.findTopicBlockAnchor).not.toHaveBeenCalled() + expect(f.submit).not.toHaveBeenCalled() + }) + + it('re-verifies the local anchor behind an equal, matching tip', async () => { + const f = fixture() + f.storage.findTopicAnchorTip.mockResolvedValue({ topic, blockHeight, tac: f.anchor.tac }) + f.storage.findTopicBlockAnchor.mockResolvedValue(f.anchor) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('matched') + expect(report.message).toBe('Topic anchor tips match') + expect(f.storage.findTopicBlockAnchor).toHaveBeenCalledWith(topic, blockHeight) + expect(f.engine.topicAnchorHeaderResolver).toHaveBeenCalledWith(blockHeight) + expect(f.requests.some(request => request.path === '/requestAdmittedList')).toBe(false) + expect(f.submit).not.toHaveBeenCalled() + }) + + it('reports historical divergence when the local tip is ahead with a different TAC', async () => { + const f = fixture() + f.storage.findTopicAnchorTip.mockResolvedValue({ + topic, + blockHeight: blockHeight + 1, + tac: 'cd'.repeat(32) + }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('diverged') + expect(report.message).toContain('Remote tip is not ahead') + // Divergence is not an anchor failure: the local tip is never re-verified. + expect(f.storage.findTopicBlockAnchor).not.toHaveBeenCalled() + expect(f.submit).not.toHaveBeenCalled() + }) + + it('fails closed when the local tip has no stored anchor', async () => { + const f = fixture() + f.storage.findTopicAnchorTip.mockResolvedValue({ topic, blockHeight, tac: f.anchor.tac }) + f.storage.findTopicBlockAnchor.mockResolvedValue(undefined) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.errorCode).toBe('BASM_INVALID_RESPONSE') + expect(report.message).toBe('Local BASM tip lacks its anchor') + expect(f.submit).not.toHaveBeenCalled() + }) + + it('fails closed when the backend reports a tip it cannot produce an anchor for', async () => { + const f = fixture() + f.storage.findTopicAnchorTip.mockResolvedValue({ topic, blockHeight, tac: f.anchor.tac }) + // A backend with tip support but no per-block anchor lookup must not have + // its equal-height tip accepted on the peer's word alone. + delete (f.storage as { findTopicBlockAnchor?: unknown }).findTopicBlockAnchor + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.errorCode).toBe('BASM_INVALID_RESPONSE') + expect(report.message).toBe('Local BASM tip lacks its anchor') + expect(f.submit).not.toHaveBeenCalled() + }) + + it('fails closed when the stored anchor disagrees with the local tip TAC', async () => { + const f = fixture() + f.storage.findTopicAnchorTip.mockResolvedValue({ topic, blockHeight, tac: f.anchor.tac }) + f.storage.findTopicBlockAnchor.mockResolvedValue({ ...f.anchor, tac: 'ef'.repeat(32) }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.errorCode).toBe('BASM_INVALID_RESPONSE') + expect(report.message).toBe('Local BASM tip lacks its anchor') + expect(f.submit).not.toHaveBeenCalled() + }) + + it('fails closed when the equal-height local anchor is not canonical', async () => { + const f = fixture() + f.storage.findTopicAnchorTip.mockResolvedValue({ topic, blockHeight, tac: f.anchor.tac }) + f.storage.findTopicBlockAnchor.mockResolvedValue(f.anchor) + f.engine.topicAnchorHeaderResolver = async height => ({ + blockHeight: height, + blockHash: 'cd'.repeat(32) + }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.message).toBe('BASM anchor block hash is not canonical') + expect(f.submit).not.toHaveBeenCalled() + }) +}) + +describe('BASM reconciliation outcome reporting', () => { + it('reports "matched" when the admitted page brings the local tip up to the remote tip', async () => { + const f = fixture() + f.storage.findTopicAnchorTip + .mockResolvedValueOnce(undefined) + .mockResolvedValue({ topic, blockHeight, tac: f.anchor.tac, blockHash }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('matched') + expect(report.fetchedTxCount).toBe(2) + expect(report.localTip).toEqual({ topic, blockHeight, tac: f.anchor.tac, blockHash }) + expect(f.submit).toHaveBeenCalledTimes(2) + }) + + it('reports "advanced" when the local tip still trails after the page is applied', async () => { + const f = fixture() + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('advanced') + expect(report.localTip).toEqual({ topic, blockHeight: -1, tac: BASM_ZERO_HASH }) + expect(f.submit).toHaveBeenCalledTimes(2) + }) + + it('stringifies a non-Error failure without inventing an error code', async () => { + const f = fixture() + f.storage.findTopicAnchorTip.mockRejectedValue('peer storage is offline') + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.message).toBe('peer storage is offline') + expect(report.errorCode).toBeUndefined() + expect(f.submit).not.toHaveBeenCalled() + }) + + it('surfaces a BASM_UNSUPPORTED storage gap as a classified sync error', async () => { + const f = fixture() + // A backend that cannot answer tip queries at all. + delete (f.storage as { findTopicAnchorTip?: unknown }).findTopicAnchorTip + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.errorCode).toBe('BASM_UNSUPPORTED') + expect(report.message).toBe('Storage does not support BASM topic anchor tips') + expect(f.submit).not.toHaveBeenCalled() + }) +}) From 85d7a581be4ac91f57e9dcb065901ea5ea8027b7 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 19:13:16 -0500 Subject: [PATCH 151/168] test(overlay): cover BASM compound-proof position binding Adds the canonical-position cases the reconciler fails closed on: a phantom duplicate at the right edge of an odd-width block is accepted but the same node is rejected once the canonical width is even, a BUMP that omits an admitted leaf is rejected before any raw transaction is fetched, a non-canonical root is rejected by the chain tracker, canonical-count assurance is recorded even when nothing needs fetching, and an anchor that vanishes from the peer blocks admission. Co-Authored-By: Claude Fable 5.1 --- .../src/__tests/BASMProofBinding.test.ts | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 packages/overlays/overlay/src/__tests/BASMProofBinding.test.ts diff --git a/packages/overlays/overlay/src/__tests/BASMProofBinding.test.ts b/packages/overlays/overlay/src/__tests/BASMProofBinding.test.ts new file mode 100644 index 000000000..5d1bcbb46 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/BASMProofBinding.test.ts @@ -0,0 +1,193 @@ +import { MerklePath } from '@bsv/sdk' +import { BASM_ZERO_HASH, computeTac } from '../BASM' +import type { TopicBlockAnchor } from '../BASM' +import { blockHash, blockHeight, fixture, hashPair, topic } from './BASMReconciliationFixture' +import type { BASMFixture } from './BASMReconciliationFixture' + +afterEach(() => jest.restoreAllMocks()) + +/** + * Re-points the fake peer at a single-block anchor whose admitted list is + * proven by `path`, and tells the chain tracker which root is canonical. + */ +function serveBlock( + f: BASMFixture, + options: { + admitted: Array<{ txid: string; blockIndex: number }> + basmRoot: string + path: MerklePath + merkleRoot: string + rawTxIndexes: number[] + blockTransactionCount?: number + } +): TopicBlockAnchor { + const anchor: TopicBlockAnchor = { + topic, + blockHeight, + blockHash, + basmRoot: options.basmRoot, + admittedCount: options.admitted.length, + tac: computeTac(BASM_ZERO_HASH, blockHash, options.basmRoot) + } + f.responses['/requestTopicAnchorTip'] = anchor + f.responses['/requestTopicAnchorRange'] = { topic, anchors: [anchor] } + f.responses['/requestAdmittedList'] = { + topic, + blockHeight, + blockHash, + admitted: options.admitted + } + f.responses['/requestCompoundMerklePath'] = { + topic, + blockHeight, + txids: options.admitted.map(item => item.txid), + merklePath: options.path.toHex() + } + f.responses['/requestRawTransactions'] = { + transactions: options.rawTxIndexes.map(index => ({ + txid: f.ids[index], + rawTx: f.transactions[index].toHex() + })), + missing: [] + } + f.tracker.isValidRootForHeight.mockImplementation( + async (candidate: string, height: number) => + candidate === options.merkleRoot && height === blockHeight + ) + f.engine.topicAnchorHeaderResolver = async height => ({ + blockHeight: height, + blockHash, + merkleRoot: options.merkleRoot, + blockTransactionCount: options.blockTransactionCount + }) + return anchor +} + +/** A three-transaction block: offset 3 is the phantom duplicate of offset 2. */ +function oddWidthBlock(f: BASMFixture): { path: MerklePath; merkleRoot: string } { + return { + path: new MerklePath(blockHeight, [ + [ + { offset: 2, hash: f.ids[2], txid: true }, + { offset: 3, duplicate: true } + ], + [{ offset: 0, hash: hashPair(f.ids[0], f.ids[1]) }] + ]), + merkleRoot: hashPair(hashPair(f.ids[0], f.ids[1]), hashPair(f.ids[2], f.ids[2])) + } +} + +describe('BASM canonical proof position validation', () => { + it('accepts a duplicate node at the right edge of an odd-width block', async () => { + const f = fixture() + const block = oddWidthBlock(f) + serveBlock(f, { + admitted: [{ txid: f.ids[2], blockIndex: 2 }], + basmRoot: f.ids[2], + path: block.path, + merkleRoot: block.merkleRoot, + rawTxIndexes: [2], + blockTransactionCount: 3 + }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('advanced') + expect(report.fetchedTxCount).toBe(1) + expect(report.positionValidation).toBe('canonical-count') + expect(f.submit).toHaveBeenCalledTimes(1) + expect(f.tracker.isValidRootForHeight).toHaveBeenCalledWith(block.merkleRoot, blockHeight) + }) + + it('rejects the same duplicate node when the canonical block width is even', async () => { + const f = fixture() + const block = oddWidthBlock(f) + serveBlock(f, { + admitted: [{ txid: f.ids[2], blockIndex: 2 }], + basmRoot: f.ids[2], + path: block.path, + merkleRoot: block.merkleRoot, + rawTxIndexes: [2], + // Four transactions leaves no odd right edge, so offset 3 is a real + // position the peer would have to prove rather than duplicate. + blockTransactionCount: 4 + }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.errorCode).toBe('BASM_INVALID_RESPONSE') + expect(report.message).toBe('BASM proof node is outside canonical block positions') + expect(f.requests.some(request => request.path === '/requestRawTransactions')).toBe(false) + expect(f.submit).not.toHaveBeenCalled() + }) + + it('rejects a compound path that silently omits one admitted transaction', async () => { + const f = fixture() + // Claims both txids in its response envelope, but the BUMP only carries a + // leaf for the first one. + f.responses['/requestCompoundMerklePath'] = { + topic, + blockHeight, + txids: f.admitted.map(item => item.txid), + merklePath: new MerklePath(blockHeight, [ + [ + { offset: 0, hash: f.ids[0] }, + { offset: 1, hash: f.ids[1], txid: true } + ], + [{ offset: 1, hash: hashPair(f.ids[2], f.ids[3]) }] + ]).toHex() + } + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.errorCode).toBe('BASM_INVALID_RESPONSE') + expect(report.message).toBe('BASM proof does not bind the admitted block index') + expect(f.requests.some(request => request.path === '/requestRawTransactions')).toBe(false) + expect(f.submit).not.toHaveBeenCalled() + }) + + it('rejects a well-formed proof whose root is not canonical for the height', async () => { + const f = fixture() + f.tracker.isValidRootForHeight.mockResolvedValue(false) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.errorCode).toBeUndefined() + expect(report.message).toBe( + `Peer supplied invalid compound Merkle path at height ${blockHeight}` + ) + expect(f.tracker.isValidRootForHeight).toHaveBeenCalledWith(f.root, blockHeight) + expect(f.requests.some(request => request.path === '/requestRawTransactions')).toBe(false) + expect(f.submit).not.toHaveBeenCalled() + }) + + it('records canonical-count assurance even when nothing needs fetching', async () => { + const f = fixture() + f.storage.findAdmittedTransactionsForBlock.mockResolvedValue([...f.admitted]) + f.engine.topicAnchorHeaderResolver = async height => ({ + blockHeight: height, + blockHash, + merkleRoot: f.root, + blockTransactionCount: 4 + }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('diverged') + expect(report.positionValidation).toBe('canonical-count') + expect(report.missingTxids).toEqual([]) + expect(f.requests.some(request => request.path === '/requestCompoundMerklePath')).toBe(true) + expect(f.requests.some(request => request.path === '/requestRawTransactions')).toBe(false) + expect(f.submit).not.toHaveBeenCalled() + }) + + it('refuses to admit when the anchor disappears from the peer before admission', async () => { + const f = fixture() + const originalFetch = f.fetchMock.getMockImplementation()! + f.fetchMock.mockImplementation(async (url, init) => { + const response = await originalFetch(url, init) + if (new URL(String(url)).pathname === '/requestRawTransactions') { + f.responses['/requestTopicAnchorRange'] = { topic, anchors: [] } + } + return response + }) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.errorCode).toBe('BASM_INVALID_RESPONSE') + expect(report.message).toBe('BASM peer anchor changed before admission') + expect(f.submit).not.toHaveBeenCalled() + }) +}) From 0ed7a9356eb15ce6cf4c8ab732fa77440bb01534 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 19:17:07 -0500 Subject: [PATCH 152/168] test(overlay): cover BASM forward-page prefix and assurance reporting Drives a multi-block fake peer to pin the anchor-page contract: a page truncated short of its requested target and a page that skips the height after the local tip are both rejected, a peer many pages ahead is followed one capped page at a time without re-downloading anchors that already match locally, and a page mixing canonical-count and encoded-offset blocks reports the weaker assurance for the whole attempt. Co-Authored-By: Claude Fable 5.1 --- .../src/__tests/BASMRangePrefix.test.ts | 252 ++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 packages/overlays/overlay/src/__tests/BASMRangePrefix.test.ts diff --git a/packages/overlays/overlay/src/__tests/BASMRangePrefix.test.ts b/packages/overlays/overlay/src/__tests/BASMRangePrefix.test.ts new file mode 100644 index 000000000..a1159bfda --- /dev/null +++ b/packages/overlays/overlay/src/__tests/BASMRangePrefix.test.ts @@ -0,0 +1,252 @@ +import { LockingScript, MerklePath, Transaction } from '@bsv/sdk' +import { Engine } from '../Engine' +import { BASM_ZERO_HASH, computeBasmRoot, computeTac } from '../BASM' +import type { + AdmittedTxRef, + TopicAnchorHeader, + TopicAnchorTip, + TopicBlockAnchor +} from '../BASM' +import type { Storage } from '../storage/Storage' + +const topic = 'tm_basm_range' +const endpoint = 'https://peer.example' + +/** A distinct, well-formed 32-byte block hash per height. */ +const hashAt = (height: number): string => + `${height.toString(16).padStart(8, '0')}${'bb'.repeat(28)}` + +interface RangeBody { + fromHeight: number + toHeight: number +} + +/** + * A fake BASM peer plus a local overlay whose stored anchors, canonical + * headers and chain tracker are supplied per test. + */ +function harness(options: { + localTip?: TopicAnchorTip + localAnchors?: TopicBlockAnchor[] + header?: (height: number) => TopicAnchorHeader + isCanonicalRoot?: (root: string, height: number) => boolean + handle: (path: string, body: Record) => unknown +}) { + const localAnchors = new Map( + (options.localAnchors ?? []).map(anchor => [anchor.blockHeight, anchor]) + ) + const storage = { + findTopicAnchorTip: jest.fn( + async (topicName: string): Promise => + topicName === topic ? options.localTip : undefined + ), + findTopicBlockAnchor: jest.fn( + async (topicName: string, height: number): Promise => + topicName === topic ? localAnchors.get(height) : undefined + ), + findAdmittedTransactionsForBlock: jest.fn(async (): Promise => []) + } + const tracker = { + currentHeight: jest.fn(async () => 5000), + isValidRootForHeight: jest.fn( + async (root: string, height: number) => options.isCanonicalRoot?.(root, height) ?? false + ) + } + const engine = new Engine({}, {}, storage as unknown as Storage, tracker) + engine.syncConfiguration = { [topic]: [endpoint] } + engine.topicAnchorHeaderResolver = jest.fn( + async (height: number) => + options.header?.(height) ?? { blockHeight: height, blockHash: hashAt(height) } + ) + engine.logger = { ...console, error: jest.fn() } + const submit = jest.spyOn(engine, 'submit').mockResolvedValue({}) + const requests: Array<{ path: string; body: Record }> = [] + jest.spyOn(globalThis, 'fetch').mockImplementation(async (url, init) => { + const path = new URL(String(url)).pathname + const body = JSON.parse(String(init?.body)) as Record + requests.push({ path, body }) + return Response.json(options.handle(path, body)) + }) + return { engine, storage, tracker, submit, requests } +} + +/** A contiguous run of zero-admission anchors chained onto `startTac`. */ +function chain(fromHeight: number, count: number, startTac: string): TopicBlockAnchor[] { + const anchors: TopicBlockAnchor[] = [] + let tac = startTac + for (let index = 0; index < count; index++) { + const blockHeight = fromHeight + index + tac = computeTac(tac, hashAt(blockHeight), BASM_ZERO_HASH) + anchors.push({ + topic, + blockHeight, + blockHash: hashAt(blockHeight), + basmRoot: BASM_ZERO_HASH, + admittedCount: 0, + tac + }) + } + return anchors +} + +const inRange = (anchors: TopicBlockAnchor[], body: Record): TopicBlockAnchor[] => { + const { fromHeight, toHeight } = body as unknown as RangeBody + return anchors.filter( + anchor => anchor.blockHeight >= fromHeight && anchor.blockHeight <= toHeight + ) +} + +afterEach(() => jest.restoreAllMocks()) + +describe('BASM forward-page prefix validation', () => { + it('rejects a page that stops short of the height it was asked for', async () => { + const [tip] = chain(1200, 1, BASM_ZERO_HASH) + const served = [...chain(201, 5, BASM_ZERO_HASH), tip] + const h = harness({ + handle: (path, body) => { + if (path === '/requestTopicAnchorTip') return tip + // A peer that silently caps every page at five anchors. + return { topic, anchors: inRange(served, body).slice(0, 5) } + } + }) + const [report] = await h.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.errorCode).toBe('BASM_INVALID_RESPONSE') + expect(report.message).toBe('BASM range omits its requested target') + // The page starts one page below the remote tip, not at genesis. + expect(h.requests.at(-1)).toEqual({ + path: '/requestTopicAnchorRange', + body: { fromHeight: 201, toHeight: 1200 } + }) + expect(h.submit).not.toHaveBeenCalled() + }) + + it('rejects a page that skips the height immediately after the local tip', async () => { + const localTac = computeTac(BASM_ZERO_HASH, hashAt(100), BASM_ZERO_HASH) + const [, tip] = chain(101, 2, localTac) + const h = harness({ + localTip: { topic, blockHeight: 100, tac: localTac }, + handle: (path, body) => { + if (path === '/requestTopicAnchorTip') return tip + // Height 101 is withheld, so the prefix cannot be chained. + return { topic, anchors: inRange([tip], body) } + } + }) + const [report] = await h.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.errorCode).toBe('BASM_INVALID_RESPONSE') + expect(report.message).toBe('BASM range omits its next height') + expect(h.submit).not.toHaveBeenCalled() + }) + + it('advances one capped page at a time when the peer is many pages ahead', async () => { + const localTac = computeTac(BASM_ZERO_HASH, hashAt(0), BASM_ZERO_HASH) + const page = chain(1, 1000, localTac) + const [tip] = chain(2000, 1, BASM_ZERO_HASH) + const h = harness({ + localTip: { topic, blockHeight: 0, tac: localTac }, + // Every anchor in the page is already stored locally with the same TAC. + localAnchors: page, + handle: (path, body) => { + if (path === '/requestTopicAnchorTip') return tip + return { topic, anchors: inRange([...page, tip], body) } + } + }) + const [report] = await h.engine.startBASMSync() + expect(report.status).toBe('advanced') + expect(report.checkedHeights).toHaveLength(1000) + expect(report.checkedHeights[0]).toBe(1) + expect(report.checkedHeights.at(-1)).toBe(1000) + expect(report.fetchedTxCount).toBe(0) + expect(report.missingTxids).toEqual([]) + // A page that ends below the remote tip is not required to reproduce it, + // and anchors already held locally are never re-downloaded. + expect( + h.requests.filter(request => request.path === '/requestTopicAnchorRange').map(r => r.body) + ).toEqual([ + { fromHeight: 2000, toHeight: 2000 }, + { fromHeight: 1, toHeight: 1000 } + ]) + expect(h.requests.some(request => request.path === '/requestAdmittedList')).toBe(false) + expect(h.submit).not.toHaveBeenCalled() + }) +}) + +describe('BASM position-assurance reporting across a page', () => { + it('keeps the weakest assurance when only some blocks bind a canonical count', async () => { + const transactions = [10, 20].map( + satoshis => + new Transaction(1, [], [{ satoshis, lockingScript: LockingScript.fromASM('OP_TRUE') }], 0) + ) + const ids = transactions.map(tx => tx.id('hex')) + const localTac = computeTac(BASM_ZERO_HASH, hashAt(99), BASM_ZERO_HASH) + const anchors: TopicBlockAnchor[] = [] + let tac = localTac + ids.forEach((txid, index) => { + const blockHeight = 100 + index + const basmRoot = computeBasmRoot([{ txid, blockIndex: 0 }]) + tac = computeTac(tac, hashAt(blockHeight), basmRoot) + anchors.push({ + topic, + blockHeight, + blockHash: hashAt(blockHeight), + basmRoot, + admittedCount: 1, + tac + }) + }) + const txidAt = (height: number): string => ids[height - 100] + const rawByTxid = new Map(ids.map((txid, index) => [txid, transactions[index].toHex()])) + const h = harness({ + localTip: { topic, blockHeight: 99, tac: localTac }, + header: height => ({ + blockHeight: height, + blockHash: hashAt(height), + merkleRoot: txidAt(height), + // Only the second block's header carries a trusted transaction count. + ...(height === 101 ? { blockTransactionCount: 1 } : {}) + }), + isCanonicalRoot: (root, height) => root === txidAt(height), + handle: (path, body) => { + if (path === '/requestTopicAnchorTip') return anchors[1] + if (path === '/requestTopicAnchorRange') return { topic, anchors: inRange(anchors, body) } + const { blockHeight, txids } = body as unknown as { + blockHeight: number + txids: string[] + } + if (path === '/requestAdmittedList') { + return { + topic, + blockHeight, + blockHash: hashAt(blockHeight), + admitted: [{ txid: txidAt(blockHeight), blockIndex: 0 }] + } + } + if (path === '/requestCompoundMerklePath') { + return { + topic, + blockHeight, + txids, + merklePath: new MerklePath(blockHeight, [ + [{ offset: 0, hash: txidAt(blockHeight), txid: true }] + ]).toHex() + } + } + return { + transactions: txids.map(txid => ({ txid, rawTx: rawByTxid.get(txid) })), + missing: [] + } + } + }) + const [report] = await h.engine.startBASMSync() + expect(report.status).toBe('advanced') + expect(report.checkedHeights).toEqual([100, 101]) + expect(report.fetchedTxCount).toBe(2) + // Height 100 could only be checked against encoded offsets, so the report + // must not claim the stronger canonical-count assurance of height 101. + expect(report.positionValidation).toBe('encoded-offset-only') + expect( + h.submit.mock.calls.map(([tagged]) => Transaction.fromBEEF(tagged.beef).id('hex')) + ).toEqual(ids) + }) +}) From 03e6c74a65c9a6d93557597f9bee824b3c50aa44 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 19:17:32 -0500 Subject: [PATCH 153/168] test(sdk): pin HTTPS lookup facilitator transport bounds Cover the cancellation and resource-limit branches of HTTPSOverlayLookupFacilitator: a pre-aborted caller signal short-circuits before any fetch, an abort racing a non-ok response reports cancellation and drains the body, JSON and octet-stream output counts are rejected above the caller's budget (including a negative varint count), atomic BEEF extraction stops once the extracted bytes outrun the response budget, and an in-flight decode observes a mid-stream cancellation. Co-Authored-By: Claude Fable 5.1 --- .../__tests/LookupResolver.transport.test.ts | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 packages/sdk/src/overlay-tools/__tests/LookupResolver.transport.test.ts diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.transport.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.transport.test.ts new file mode 100644 index 000000000..d31c59394 --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.transport.test.ts @@ -0,0 +1,216 @@ +import { + HTTPSOverlayLookupFacilitator, + LookupResourceLimitError, + type LookupAnswer, + type LookupFacilitatorAnswer +} from '../LookupResolver' +import { Transaction } from '../../transaction/index' +import { LockingScript } from '../../script/index' + +const question = { service: 'ls_transport', query: { id: 1 } } +const host = 'https://transport.example' + +const jsonResponse = (body: unknown): Response => + new Response(JSON.stringify(body), { headers: { 'content-type': 'application/json' } }) + +const octetResponse = (payload: Uint8Array): Response => + new Response(payload, { headers: { 'content-type': 'application/octet-stream' } }) + +/** + * Structurally parseable receipt. Parsing BEEF here makes no chain-validity + * claim; these tests only exercise the transport's own byte and count bounds. + */ +function transaction(scriptBytes: number, satoshis = 1): Transaction { + return new Transaction( + 1, + [], + [{ lockingScript: LockingScript.fromHex('00'.repeat(scriptBytes)), satoshis }], + 0 + ) +} + +/** + * Aggregated octet-stream wire format: varint outpoint count, then per outpoint + * a 32-byte txid, a varint output index and a varint-prefixed context, followed + * by the shared BEEF. `repeats` outpoints all reference the same transaction. + */ +function octetPayload(tx: Transaction, repeats: number): Buffer { + const txid = Buffer.from(tx.id('hex'), 'hex') + const outpoints = Array.from({ length: repeats }, (_unused, index) => + Buffer.concat([txid, Buffer.from([index]), Buffer.from([0x00])]) + ) + return Buffer.concat([Buffer.from([repeats]), ...outpoints, Buffer.from(tx.toBEEF())]) +} + +function outputsOf(answer: LookupFacilitatorAnswer): LookupAnswer['outputs'] { + if (answer.type !== 'output-list') throw new Error('expected an output-list answer') + return answer.outputs +} + +async function caught(work: Promise): Promise { + return await work.then( + () => { + throw new Error('expected the lookup to reject') + }, + (error: unknown) => error + ) +} + +describe('HTTPSOverlayLookupFacilitator bounded transport', () => { + it('refuses to issue a request when the caller signal is already aborted', async () => { + const fetchClient = jest.fn() + const facilitator = new HTTPSOverlayLookupFacilitator( + fetchClient as unknown as typeof fetch, + true + ) + const controller = new AbortController() + controller.abort() + + const error = await caught(facilitator.lookup(host, question, 2000, controller.signal)) + + expect(error).toBeInstanceOf(Error) + expect((error as Error).name).toBe('AbortError') + expect((error as Error).message).toBe('Lookup cancelled') + expect(fetchClient).not.toHaveBeenCalled() + }) + + it('reports cancellation, not an HTTP failure, when the caller aborts as the response arrives', async () => { + const controller = new AbortController() + let issued: Response | undefined + const fetchClient = jest.fn(async () => { + controller.abort() + issued = new Response('service unavailable', { status: 503, statusText: 'Unavailable' }) + return issued + }) + const facilitator = new HTTPSOverlayLookupFacilitator( + fetchClient as unknown as typeof fetch, + true + ) + + const error = await caught(facilitator.lookup(host, question, 2000, controller.signal)) + // Let the in-flight request settle so its body cleanup is observable. + await new Promise(resolve => setTimeout(resolve, 0)) + + expect((error as Error).name).toBe('AbortError') + expect((error as Error).message).toBe('Lookup cancelled') + // A cancelled request must not leave the response body undrained. + expect(issued?.bodyUsed).toBe(true) + }) + + it('rejects a JSON output list longer than the requested output budget', async () => { + const answer = { + type: 'output-list', + outputs: [ + { beef: transaction(1, 1).toBEEF(), outputIndex: 0 }, + { beef: transaction(1, 2).toBEEF(), outputIndex: 0 } + ] + } + const facilitator = new HTTPSOverlayLookupFacilitator( + jest.fn(async () => jsonResponse(answer)) as unknown as typeof fetch, + true + ) + + const error = await caught( + facilitator.lookup(host, question, 2000, undefined, { maxOutputs: 1 }) + ) + expect(error).toBeInstanceOf(LookupResourceLimitError) + expect((error as LookupResourceLimitError).limit).toBe('maxOutputs') + + // The bound is inclusive: exactly maxOutputs is still accepted. + const accepted = await facilitator.lookup(host, question, 2000, undefined, { maxOutputs: 2 }) + expect(outputsOf(accepted)).toHaveLength(2) + }) + + it('rejects an octet-stream outpoint count that is negative or over the output budget', async () => { + const tx = transaction(4) + const overBudget = new HTTPSOverlayLookupFacilitator( + jest.fn(async () => octetResponse(octetPayload(tx, 3))) as unknown as typeof fetch, + true + ) + const tooMany = await caught( + overBudget.lookup(host, question, 2000, undefined, { maxOutputs: 2 }) + ) + expect(tooMany).toBeInstanceOf(LookupResourceLimitError) + expect((tooMany as LookupResourceLimitError).limit).toBe('maxOutputs') + + // 0xff + eight 0xff bytes decodes as -1: a count that must never be trusted. + const negativeCount = Buffer.concat([ + Buffer.from([0xff]), + Buffer.alloc(8, 0xff), + Buffer.from(tx.toBEEF()) + ]) + const negative = new HTTPSOverlayLookupFacilitator( + jest.fn(async () => octetResponse(negativeCount)) as unknown as typeof fetch, + true + ) + const malformed = await caught( + negative.lookup(host, question, 2000, undefined, { maxOutputs: 64 }) + ) + expect(malformed).toBeInstanceOf(LookupResourceLimitError) + expect((malformed as LookupResourceLimitError).limit).toBe('maxOutputs') + }) + + it('stops octet-stream extraction once the extracted bytes exceed the response budget', async () => { + const tx = transaction(400) + const payload = octetPayload(tx, 3) + const beefBytes = tx.toBEEF().length + // Three outpoints on one transaction extract three atomic BEEF copies, so + // the retained total outruns the wire length the reader already accepted. + expect(2 * beefBytes).toBeGreaterThan(payload.length + 1) + const facilitator = new HTTPSOverlayLookupFacilitator( + jest.fn(async () => octetResponse(payload)) as unknown as typeof fetch, + true + ) + + const error = await caught( + facilitator.lookup(host, question, 2000, undefined, { + maxResponseBytes: payload.length + 1, + maxOutputs: 8 + }) + ) + expect(error).toBeInstanceOf(LookupResourceLimitError) + expect((error as LookupResourceLimitError).limit).toBe('maxResponseBytes') + + const accepted = await facilitator.lookup(host, question, 2000, undefined, { + maxResponseBytes: 4 * beefBytes, + maxOutputs: 8 + }) + const outputs = outputsOf(accepted) + expect(outputs).toHaveLength(3) + expect(outputs.map(output => output.txid)).toEqual([tx.id('hex'), tx.id('hex'), tx.id('hex')]) + }) + + it('abandons octet-stream extraction when the caller cancels mid-decode', async () => { + const controller = new AbortController() + const tx = transaction(8) + const payload = octetPayload(tx, 60) + let scheduled = false + const facilitator = new HTTPSOverlayLookupFacilitator( + jest.fn(async () => octetResponse(payload)) as unknown as typeof fetch, + true + ) + + const startedAt = Date.now() + const error = await caught( + facilitator.lookup(host, question, 5000, controller.signal, { + maxResponseBytes: 1_000_000, + maxOutputs: 128, + // Cancel once the transport has begun reporting bytes: extraction + // yields to the event loop between outputs and must observe the abort. + consumeBytes: () => { + if (scheduled) return + scheduled = true + setTimeout(() => controller.abort(), 0) + } + }) + ) + // The abandoned decode is detached from the caller-facing promise; give it + // an event-loop turn so its own cancellation check runs before teardown. + await new Promise(resolve => setTimeout(resolve, 30)) + + expect((error as Error).name).toBe('AbortError') + expect((error as Error).message).toBe('Lookup cancelled') + // Cancellation settles the request rather than waiting out the 5s deadline. + expect(Date.now() - startedAt).toBeLessThan(2000) + }) +}) From dbb3947e977a5d55d6410d6ff5eee8577a96a581 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 19:20:37 -0500 Subject: [PATCH 154/168] style(overlay): prettier-format the new BASM test suites Co-Authored-By: Claude Fable 5.1 --- .../src/__tests/BASMProviderGuards.test.ts | 5 +---- .../overlay/src/__tests/BASMRangePrefix.test.ts | 17 +++++++---------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/overlays/overlay/src/__tests/BASMProviderGuards.test.ts b/packages/overlays/overlay/src/__tests/BASMProviderGuards.test.ts index bcf89dd42..14aebff6d 100644 --- a/packages/overlays/overlay/src/__tests/BASMProviderGuards.test.ts +++ b/packages/overlays/overlay/src/__tests/BASMProviderGuards.test.ts @@ -8,10 +8,7 @@ const txid = 'ab'.repeat(32) const engineWithout = (methods: Record = {}): Engine => new Engine({}, {}, methods as unknown as Storage, 'scripts only') -const expectUnsupported = async ( - operation: Promise, - message: string -): Promise => { +const expectUnsupported = async (operation: Promise, message: string): Promise => { await expect(operation).rejects.toThrow(TypeError) await expect(operation).rejects.toThrow(message) await expect(operation).rejects.toMatchObject({ code: 'BASM_UNSUPPORTED' }) diff --git a/packages/overlays/overlay/src/__tests/BASMRangePrefix.test.ts b/packages/overlays/overlay/src/__tests/BASMRangePrefix.test.ts index a1159bfda..a3b270ebf 100644 --- a/packages/overlays/overlay/src/__tests/BASMRangePrefix.test.ts +++ b/packages/overlays/overlay/src/__tests/BASMRangePrefix.test.ts @@ -1,12 +1,7 @@ import { LockingScript, MerklePath, Transaction } from '@bsv/sdk' import { Engine } from '../Engine' import { BASM_ZERO_HASH, computeBasmRoot, computeTac } from '../BASM' -import type { - AdmittedTxRef, - TopicAnchorHeader, - TopicAnchorTip, - TopicBlockAnchor -} from '../BASM' +import type { AdmittedTxRef, TopicAnchorHeader, TopicAnchorTip, TopicBlockAnchor } from '../BASM' import type { Storage } from '../storage/Storage' const topic = 'tm_basm_range' @@ -36,9 +31,8 @@ function harness(options: { (options.localAnchors ?? []).map(anchor => [anchor.blockHeight, anchor]) ) const storage = { - findTopicAnchorTip: jest.fn( - async (topicName: string): Promise => - topicName === topic ? options.localTip : undefined + findTopicAnchorTip: jest.fn(async (topicName: string): Promise => + topicName === topic ? options.localTip : undefined ), findTopicBlockAnchor: jest.fn( async (topicName: string, height: number): Promise => @@ -89,7 +83,10 @@ function chain(fromHeight: number, count: number, startTac: string): TopicBlockA return anchors } -const inRange = (anchors: TopicBlockAnchor[], body: Record): TopicBlockAnchor[] => { +const inRange = ( + anchors: TopicBlockAnchor[], + body: Record +): TopicBlockAnchor[] => { const { fromHeight, toHeight } = body as unknown as RangeBody return anchors.filter( anchor => anchor.blockHeight >= fromHeight && anchor.blockHeight <= toHeight From f4594c104dd8c5ddd111e2cc8e1383cdeb5daba3 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 19:20:42 -0500 Subject: [PATCH 155/168] test(sdk): pin lookup query resource, cancellation and host-policy bounds Exercise the uncovered fail-closed branches of LookupResolver.query$: aggregation stopping at maxOutputs across hosts, the evidenceLimits shorthand defaulting its byte budget, the bounded candidate scan (scan window, malformed entries, maxHosts), per-source quotas for additional hosts, SLAP discovery refusing bytes that would breach maxTotalBytes, byte reports arriving after cancellation, cancellation dropping an in-flight peer answer and skipping queued hosts, idempotent double cancellation, the concurrent-query ceiling, over-budget answers being dropped without blaming the host, the maxTrackers budget, advertisement-map eviction, and a broader host cache surviving a tighter rediscovery. Co-Authored-By: Claude Fable 5.1 --- .../__tests/LookupResolver.limits.test.ts | 567 ++++++++++++++++++ 1 file changed, 567 insertions(+) create mode 100644 packages/sdk/src/overlay-tools/__tests/LookupResolver.limits.test.ts diff --git a/packages/sdk/src/overlay-tools/__tests/LookupResolver.limits.test.ts b/packages/sdk/src/overlay-tools/__tests/LookupResolver.limits.test.ts new file mode 100644 index 000000000..6ebb95d1c --- /dev/null +++ b/packages/sdk/src/overlay-tools/__tests/LookupResolver.limits.test.ts @@ -0,0 +1,567 @@ +import LookupResolver, { + DEFAULT_LOOKUP_LIMITS, + LookupResourceLimitError, + type LookupAnswer, + type LookupAnswerProgress, + type LookupEvidenceEvent, + type LookupFacilitatorAnswer, + type LookupQuestion, + type LookupRequestOptions, + type UnreachableHostInfo +} from '../LookupResolver' +import { getOverlayHostReputationTracker } from '../HostReputationTracker' +import OverlayAdminTokenTemplate from '../OverlayAdminTokenTemplate' +import { CompletedProtoWallet } from '../../auth/certificates/__tests/CompletedProtoWallet' +import { PrivateKey } from '../../primitives/index' +import { LockingScript } from '../../script/index' +import { Transaction } from '../../transaction/index' + +const service = 'ls_limits' +const question: LookupQuestion = { service, query: { id: 1 } } + +type LookupOutput = LookupAnswer['outputs'][number] + +/** Structurally parseable receipt; no chain-validity claim is made here. */ +function receipt(satoshis: number): LookupOutput { + const tx = new Transaction(1, [], [{ lockingScript: LockingScript.fromHex('88'), satoshis }], 0) + return { beef: tx.toBEEF(), outputIndex: 0 } +} + +/** A SLAP advertisement naming `domain` as a host for `advertised`. */ +async function slapAdvertisement( + scalar: number, + domain: string, + advertised: string +): Promise { + const wallet = new CompletedProtoWallet(new PrivateKey(scalar)) + const template = new OverlayAdminTokenTemplate(wallet) + const lockingScript = await template.lock('SLAP', domain, advertised) + const tx = new Transaction(1, [], [{ lockingScript, satoshis: 1 }], 0) + return { beef: tx.toBEEF(), outputIndex: 0 } +} + +interface Deferred { + promise: Promise + resolve: (value: T) => void +} + +function deferred(): Deferred { + let resolve!: (value: T) => void + const promise = new Promise(resolvePromise => { + resolve = resolvePromise + }) + return { promise, resolve } +} + +async function collect( + progress: AsyncIterable +): Promise { + const emissions: LookupAnswerProgress[] = [] + for await (const emission of progress) emissions.push(emission) + return emissions +} + +async function finalEmission( + progress: AsyncIterable +): Promise { + const emissions = await collect(progress) + const last = emissions.at(-1) + if (last === undefined) throw new Error('expected at least one emission') + return last +} + +async function caught(work: Promise): Promise { + return await work.then( + () => { + throw new Error('expected the query to reject') + }, + (error: unknown) => error + ) +} + +/** One macrotask turn: every pending microtask continuation has run. */ +async function eventLoopTurn(): Promise { + await new Promise(resolve => setTimeout(resolve, 0)) +} + +describe('LookupResolver query resource bounds', () => { + beforeEach(() => { + getOverlayHostReputationTracker().reset() + }) + + afterEach(() => { + getOverlayHostReputationTracker().reset() + }) + + it('stops merging at maxOutputs once a second host contributes a new outpoint', async () => { + const answers: Record = { + 'https://first.example': { type: 'output-list', outputs: [receipt(1)] }, + 'https://second.example': { type: 'output-list', outputs: [receipt(2)] } + } + const resolver = new LookupResolver({ + hostOverrides: { [service]: Object.keys(answers) }, + limits: { maxOutputs: 1 }, + facilitator: { lookup: async host => answers[host] } + }) + + const { answer, progress } = await resolver.queryDetailed(question) + + expect(answer.outputs).toHaveLength(1) + expect(progress.successfulHosts).toBe(2) + expect(progress.limitsHit).toContain('maxOutputs') + expect(progress.terminalReason).toBe('resource-limit') + }) + + it('defaults the evidence byte budget when only an output count is supplied', async () => { + const events: LookupEvidenceEvent[] = [] + const first = receipt(1) + const resolver = new LookupResolver({ + hostOverrides: { [service]: ['https://evidence.example'] }, + facilitator: { + lookup: async () => ({ type: 'output-list', outputs: [first, receipt(2)] }) + } + }) + + const { progress } = await resolver.queryDetailed(question, undefined, { + evidenceLimits: { maxOutputs: 1 }, + onEvidence: event => { + events.push(event) + } + }) + + expect(events.filter(event => event.type === 'output')).toHaveLength(1) + expect(events.at(-1)).toEqual({ type: 'limit' }) + expect(progress.limitsHit).toContain('maxEvidenceOutputs') + // The default byte budget is generous: the single receipt was admitted whole. + expect(progress.evidenceBytes).toBe(first.beef.length) + // Evidence intake is additive; legacy aggregation still merged both outputs. + expect(progress.outputs).toHaveLength(2) + }) + + it('bounds the candidate scan and skips malformed and over-budget host entries', async () => { + const queried: string[] = [] + const resolver = new LookupResolver({ + hostOverrides: { + [service]: [ + 'not a url', + 'https://h1.example', + 'https://h2.example', + 'https://h3.example', + 'https://h4.example', + 'https://h5.example' + ] + }, + limits: { maxHosts: 1 }, + facilitator: { + lookup: async host => { + queried.push(host) + return { type: 'output-list', outputs: [receipt(1)] } + } + } + }) + + const { answer, progress } = await resolver.queryDetailed(question) + + expect(queried).toEqual(['https://h1.example']) + expect(answer.outputs).toHaveLength(1) + expect(progress.discoveredHosts).toBe(1) + // 2 beyond the maxHosts * 4 scan window, 1 unparseable, 2 past maxHosts. + expect(progress.skippedHosts).toBe(5) + expect(progress.limitsHit).toContain('maxHosts') + }) + + it('cancelling from the first limit notification leaves every host unqueried', async () => { + const controller = new AbortController() + const queried: string[] = [] + const events: LookupEvidenceEvent[] = [] + const resolver = new LookupResolver({ + hostOverrides: { [service]: ['https://one.example', 'https://two.example'] }, + additionalHosts: { [service]: ['https://three.example'] }, + limits: { maxHosts: 1 }, + facilitator: { + lookup: async host => { + queried.push(host) + return { type: 'output-list', outputs: [] } + } + } + }) + + const error = await caught( + resolver.query(question, undefined, { + signal: controller.signal, + onEvidence: event => { + events.push(event) + if (event.type === 'limit') controller.abort() + } + }) + ) + + expect((error as Error).name).toBe('AbortError') + expect(queried).toEqual([]) + // Exactly one limit notification, even though cancellation records its own. + expect(events).toEqual([{ type: 'limit' }]) + }) + + it('reserves a per-source quota for additional hosts while discovery is refreshed', async () => { + const queried: string[] = [] + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: ['https://tracker.example'], + additionalHosts: { [service]: ['https://add1.example', 'https://add2.example'] }, + limits: { maxHosts: 2, maxTrackers: 1 }, + facilitator: { + lookup: async (host, asked) => { + queried.push(host) + if (asked.service === 'ls_slap') return { type: 'output-list', outputs: [] } + return { type: 'output-list', outputs: [receipt(1)] } + } + } + }) + + const { answer, progress } = await resolver.queryDetailed(question) + + expect(queried).toEqual(['https://add1.example', 'https://tracker.example']) + expect(answer.outputs).toHaveLength(1) + expect(progress.discoveredHosts).toBe(1) + expect(progress.skippedHosts).toBe(1) + expect(progress.limitsHit).toContain('maxHosts') + }) + + it('cancelling on the quota limit releases discovery before any tracker or host is contacted', async () => { + const controller = new AbortController() + const queried: string[] = [] + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: ['https://tracker.example'], + additionalHosts: { [service]: ['https://add1.example', 'https://add2.example'] }, + limits: { maxHosts: 2, maxTrackers: 1 }, + facilitator: { + lookup: async host => { + queried.push(host) + return { type: 'output-list', outputs: [] } + } + } + }) + + const error = await caught( + resolver.query(question, undefined, { + signal: controller.signal, + onEvidence: event => { + if (event.type === 'limit') controller.abort() + } + }) + ) + await eventLoopTurn() + + expect((error as Error).name).toBe('AbortError') + // The additional host was already dispatched and the SLAP refresh was about + // to subscribe; cancellation must reach both before either sends a request. + expect(queried).toEqual([]) + }) + + it('fails closed when SLAP discovery bytes would exceed the aggregate budget', async () => { + const hostCharged = deferred() + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: ['https://tracker.example'], + additionalHosts: { [service]: ['https://add.example'] }, + limits: { maxHosts: 4, maxTrackers: 1, maxTotalBytes: 100 }, + facilitator: { + lookup: async ( + _host: string, + asked: LookupQuestion, + _timeout?: number, + _signal?: AbortSignal, + options?: LookupRequestOptions + ) => { + if (asked.service === 'ls_slap') { + await hostCharged.promise + await eventLoopTurn() + options?.consumeBytes?.(30) + return { type: 'output-list', outputs: [] } + } + options?.consumeBytes?.(80) + hostCharged.resolve() + return { type: 'output-list', outputs: [] } + } + } + }) + + const { progress } = await resolver.queryDetailed(question) + + expect(progress.hostCount).toBe(1) + expect(progress.successfulHosts).toBe(1) + // 80 host bytes were accepted; the 30 discovery bytes that would have + // breached maxTotalBytes are refused and never credited. + expect(progress.receivedBytes).toBe(80) + expect(progress.limitsHit).toContain('maxTotalBytes') + expect(progress.terminalReason).toBe('resource-limit') + expect(progress.discoveryComplete).toBe(false) + }) + + it('refuses byte reports that arrive after the query was cancelled', async () => { + const controller = new AbortController() + const resolver = new LookupResolver({ + hostOverrides: { [service]: ['https://late.example'] }, + limits: { maxTotalBytes: 1024 }, + facilitator: { + lookup: async ( + _host: string, + _asked: LookupQuestion, + _timeout?: number, + _signal?: AbortSignal, + options?: LookupRequestOptions + ) => { + options?.consumeBytes?.(4) + controller.abort() + options?.consumeBytes?.(4) + return { type: 'output-list', outputs: [] } + } + } + }) + + const final = await finalEmission( + resolver.query$(question, undefined, { signal: controller.signal }) + ) + + expect(final.terminalReason).toBe('cancelled') + expect(final.receivedBytes).toBe(4) + }) + + it('cancelling mid-flight drops the in-flight peer answer and skips queued hosts', async () => { + const controller = new AbortController() + const queried: string[] = [] + const hosts = ['https://q1.example', 'https://q2.example', 'https://q3.example'] + const resolver = new LookupResolver({ + hostOverrides: { [service]: hosts }, + limits: { maxHosts: 3, hostConcurrency: 2 }, + facilitator: { + lookup: async host => { + queried.push(host) + return { type: 'output-list', outputs: [receipt(hosts.indexOf(host) + 1)] } + } + } + }) + + const final = await finalEmission( + resolver.query$(question, undefined, { + signal: controller.signal, + onEvidence: event => { + if (event.type === 'output') controller.abort() + } + }) + ) + + expect(queried).toEqual(['https://q1.example', 'https://q2.example']) + expect(final.terminalReason).toBe('cancelled') + expect(final.hostCount).toBe(2) + // The third host never left the queue, and neither in-flight answer was + // aggregated: a cancelled attempt never answered the question. + expect(final.skippedHosts).toBe(1) + expect(final.successfulHosts).toBe(0) + expect(final.outputs).toEqual([]) + }) + + it('treats a second cancellation of the same query as a no-op', async () => { + const controller = new AbortController() + const events: LookupEvidenceEvent[] = [] + const resolver = new LookupResolver({ + hostOverrides: { [service]: ['https://stalled.example'] }, + facilitator: { + lookup: async () => await new Promise(() => {}) + } + }) + + const iterator = resolver + .query$(question, undefined, { + signal: controller.signal, + softTimeoutMs: 0, + graceMs: 0, + onEvidence: event => { + events.push(event) + } + }) + [Symbol.asyncIterator]() + + const first = await iterator.next() + expect(first.done).toBe(false) + expect(first.value.isFinal).toBe(false) + expect(first.value.hostCount).toBe(1) + + controller.abort() + // Breaking the iterator cancels a second time through the iterator signal. + await iterator.return?.(undefined) + + expect(events).toEqual([{ type: 'limit' }]) + }) + + it('refuses a new query once the concurrent query ceiling is reached', async () => { + const gate = deferred() + const resolver = new LookupResolver({ + hostOverrides: { [service]: ['https://capped.example'] }, + facilitator: { lookup: async () => await gate.promise } + }) + + const iterators = Array.from({ length: 128 }, () => + resolver.query$(question)[Symbol.asyncIterator]() + ) + const pending = iterators.map(async iterator => await iterator.next()) + await eventLoopTurn() + + const error = await caught(resolver.query(question)) + expect(error).toBeInstanceOf(LookupResourceLimitError) + expect((error as LookupResourceLimitError).limit).toBe('activeQueries') + + gate.resolve({ type: 'output-list', outputs: [] }) + await Promise.all(pending) + await Promise.all( + iterators.map(async iterator => { + await iterator.return?.(undefined) + }) + ) + + // Every finished query released its slot. + const released = await resolver.queryDetailed(question) + expect(released.progress.hostCount).toBe(1) + }) + + it('drops an answer whose receipts exceed maxResponseBytes without blaming the host', async () => { + const host = 'https://oversized.example' + const resolver = new LookupResolver({ + hostOverrides: { [service]: [host] }, + limits: { maxResponseBytes: 16 }, + facilitator: { lookup: async () => ({ type: 'output-list', outputs: [receipt(1)] }) } + }) + + const { answer, progress } = await resolver.queryDetailed(question) + + expect(answer.outputs).toEqual([]) + expect(progress.limitsHit).toEqual(['maxResponseBytes']) + expect(progress.terminalReason).toBe('resource-limit') + // A client-side budget rejection is not an availability failure. + expect(progress.failedHosts).toBe(0) + expect(progress.rejectedHosts).toBe(0) + expect(getOverlayHostReputationTracker().snapshot(host)?.totalFailures).toBe(0) + }) + + it('drops an answer with more outputs than maxOutputs instead of truncating it', async () => { + const host = 'https://overcounted.example' + const resolver = new LookupResolver({ + hostOverrides: { [service]: [host] }, + limits: { maxOutputs: 1 }, + facilitator: { + lookup: async () => ({ type: 'output-list', outputs: [receipt(1), receipt(2)] }) + } + }) + + const { answer, progress } = await resolver.queryDetailed(question) + + expect(answer.outputs).toEqual([]) + expect(progress.limitsHit).toEqual(['maxOutputs']) + expect(progress.failedHosts).toBe(0) + expect(getOverlayHostReputationTracker().snapshot(host)?.totalFailures).toBe(0) + }) + + it('queries only the budgeted number of SLAP trackers and names the limit it hit', async () => { + const trackers = ['https://t1.example', 'https://t2.example'] + const queried: string[] = [] + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: trackers, + limits: { maxTrackers: 1 }, + facilitator: { + lookup: async host => { + queried.push(host) + return { type: 'output-list', outputs: [] } + } + } + }) + + const error = await caught(resolver.query(question)) + + expect(queried).toEqual(['https://t1.example']) + // A budget exhausted during discovery keeps its own error rather than + // borrowing the no-competent-hosts message. + expect(error).toBeInstanceOf(LookupResourceLimitError) + expect((error as LookupResourceLimitError).limit).toBe('maxTrackers') + }) + + it('evicts the oldest SLAP attribution once the advertisement map is full', async () => { + const tracker = 'https://ad-tracker.example' + const advertisements = await Promise.all([ + slapAdvertisement(11, 'https://adv1.example', service), + slapAdvertisement(12, 'https://adv2.example', service), + slapAdvertisement(13, 'https://adv3.example', service) + ]) + const unreachable: UnreachableHostInfo[] = [] + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: [tracker], + cache: { hostsMaxEntries: 1 }, + limits: { maxHosts: 2, maxTrackers: 1 }, + facilitator: { + lookup: async (_host, asked) => { + if (asked.service === 'ls_slap') { + return { type: 'output-list', outputs: advertisements } + } + throw new Error('connection refused') + } + } + }) + + const { progress } = await resolver.queryDetailed(question, undefined, { + onUnreachableHost: info => { + unreachable.push(info) + } + }) + + expect(progress.failedHosts).toBe(2) + const attribution = new Map(unreachable.map(info => [info.host, info.advertisedBy])) + expect(attribution.size).toBe(2) + expect(attribution.get('https://adv2.example')).toBe(tracker) + // adv1 was evicted when adv3's attribution arrived, so it reports no tracker. + expect(attribution.has('https://adv1.example')).toBe(true) + expect(attribution.get('https://adv1.example')).toBeUndefined() + }) + + it('keeps a still-fresh broader host cache when a tighter query rediscovers', async () => { + const tracker = 'https://cache-tracker.example' + const broad = await slapAdvertisement(21, 'https://cached.example', service) + const tight = await slapAdvertisement(22, 'https://rediscovered.example', service) + let trackerCalls = 0 + const resolver = new LookupResolver({ + networkPreset: 'mainnet', + slapTrackers: [tracker], + limits: { maxTrackers: 1 }, + facilitator: { + lookup: async (_host, asked) => { + if (asked.service === 'ls_slap') { + trackerCalls++ + return { type: 'output-list', outputs: [trackerCalls === 1 ? broad : tight] } + } + // An immediate-backoff failure, so the cached host stops being + // available and the tighter query must refresh discovery. + throw new Error('Failed to fetch') + } + } + }) + + const first = await resolver.queryDetailed(question) + expect(first.progress.failedHosts).toBe(1) + + const second = await resolver.queryDetailed(question, undefined, { + limits: { maxOutputs: 8 } + }) + expect(trackerCalls).toBe(2) + expect(second.progress.failedHosts).toBe(1) + + const cache = ( + resolver as unknown as { + hostsCache: Map + } + ).hostsCache + const entry = cache.get(service) + expect(entry?.hosts).toEqual(['https://cached.example']) + expect(entry?.maxOutputs).toBe(DEFAULT_LOOKUP_LIMITS.maxOutputs) + }) +}) From 0e52687ec194bc58535177d0760bab48e907b06d Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 20:01:12 -0500 Subject: [PATCH 156/168] test(overlay): cover MongoAdmissionPlan STEAK-binding fail-closed branches Adds cases for isBoundSteak rejecting a non-object STEAK entry, an entry missing outputsToAdmit, a STEAK record that omits the plan's own decision topic, and an output index that fails to parse while binding STEAK; also covers validateAdmissionPlan rejecting a non-canonical expected-history fence. Two branches in samePayload's optional chaining are confirmed unreachable (they require a payload ref with an undefined digest, which the type system and every real caller already rule out). Co-Authored-By: Claude Fable 5.1 --- .../__tests/mongo/MongoAdmissionPlan.test.ts | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts index f1cdfc9bb..03fec866d 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionPlan.test.ts @@ -7,6 +7,7 @@ import { AdmissionRejectedError, admissionPlanPayloads, admissionReceiptFor, + isBoundSteak, lookupOutboxIntents, propagationOutboxIntents, rejectAdmission, @@ -166,6 +167,16 @@ describe('Mongo admission plan validation', () => { expect(validateAdmissionPlan(kind)).toBe('invalid-plan') }) + test('rejects a decision whose expected history fence is not canonical uint64', () => { + const badEpoch = clone(admissionPlan('history-epoch-format')) + badEpoch.decisions[0].expectedHistory.chainEpoch = 'not-a-number' + expect(validateAdmissionPlan(badEpoch)).toBe('invalid-plan') + + const badGeneration = clone(admissionPlan('history-generation-format')) + badGeneration.decisions[0].expectedHistory.topicHistoryGeneration = '01' + expect(validateAdmissionPlan(badGeneration)).toBe('invalid-plan') + }) + test('rejects malformed spends, edges, outputs, and applied history', () => { const outputIndex = clone(admissionPlan('output-index')) outputIndex.decisions[0].spends[0].outpoint.outputIndex = '4294967296' @@ -266,4 +277,46 @@ describe('Mongo admission plan validation', () => { expect(() => rejectAdmission('digest-mismatch')).toThrow(AdmissionRejectedError) expect(() => rejectAdmission('spend-conflict')).toThrow('spend-conflict') }) + + test('isBoundSteak rejects a non-object entry and a topic missing from the STEAK record', () => { + const plan = admissionPlan('steak-entry-shape') + + // A STEAK record whose value for a real topic key is not an + // outputsToAdmit-shaped object at all (here a bare number) must fail + // isSteakEntry's own `typeof`/array guard rather than being coerced. + const scalarEntry = clone(plan) + scalarEntry.steak = JSON.stringify({ tm_contract: 5 }) + expect(isBoundSteak(scalarEntry)).toBe(false) + expect(validateAdmissionPlan(scalarEntry)).toBe('invalid-plan') + + // An object entry that is missing outputsToAdmit entirely (as opposed to + // having the wrong type for coinsToRetain/coinsRemoved) must also fail. + const missingOutputsToAdmit = clone(plan) + missingOutputsToAdmit.steak = JSON.stringify({ tm_contract: {} }) + expect(isBoundSteak(missingOutputsToAdmit)).toBe(false) + expect(validateAdmissionPlan(missingOutputsToAdmit)).toBe('invalid-plan') + + // A STEAK record that never mentions the plan's own decision topic must + // fail the per-decision binding check even though every key it does have + // is individually well-shaped. + const missingTopic = clone(plan) + missingTopic.steak = JSON.stringify({ some_other_topic: { outputsToAdmit: [] } }) + expect(isBoundSteak(missingTopic)).toBe(false) + expect(validateAdmissionPlan(missingTopic)).toBe('invalid-plan') + }) + + test('isBoundSteak rejects an output index that cannot be parsed while binding STEAK', () => { + const overflow = clone(admissionPlan('steak-output-overflow')) + // The STEAK entry itself stays well-shaped and in range so isSteakEntry + // accepts it; the decision's own output carries the out-of-range index, + // so isBoundSteak's decisions.every must reach the parseStorageOutputIndex + // call for the *decision's* outputs and fail closed via its try/catch + // rather than throwing out of validateAdmissionPlan. + overflow.decisions[0].outputs[0].outputIndex = '4294967296' + overflow.steak = JSON.stringify({ + tm_contract: { outputsToAdmit: [0], coinsToRetain: [], coinsRemoved: [] } + }) + expect(isBoundSteak(overflow)).toBe(false) + expect(validateAdmissionPlan(overflow)).toBe('invalid-plan') + }) }) From 3f5c3f2a7452151dd81bfe6bbf7d80eb48746b51 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 20:01:20 -0500 Subject: [PATCH 157/168] test(overlay): cover EngineAdmission STEAK/coin-recovery branches and Engine's no-callback path EngineAdmission.test.ts: covers the top-level STEAK pass carrying a caller-supplied coinsRemoved through unchanged for a topic the accepted loop never re-derives, and classifyCoins recovering a previous txid from sourceTransaction.id() when sourceTXID is absent. Engine.admission.test.ts: covers acknowledgeOverlayAdmission's early return when no topic was accepted and no onSteakReady callback was supplied, so the optional-callback branch is exercised without throwing. Co-Authored-By: Claude Fable 5.1 --- .../src/__tests/Engine.admission.test.ts | 16 +++++ .../src/__tests/EngineAdmission.test.ts | 62 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/packages/overlays/overlay/src/__tests/Engine.admission.test.ts b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts index 9dad99fb5..355e97001 100644 --- a/packages/overlays/overlay/src/__tests/Engine.admission.test.ts +++ b/packages/overlays/overlay/src/__tests/Engine.admission.test.ts @@ -257,6 +257,22 @@ describe('Engine admission submit', () => { expect(onReady).toHaveBeenCalledWith(steak) }) + test('does not throw when no topic is accepted and no onSteakReady callback is supplied', async () => { + mockTopicManager.identifyAdmissibleOutputs = jest.fn(async () => ({ + outputsToAdmit: [], + coinsToRetain: [] + })) + const engine = new Engine( + { Hello: mockTopicManager }, + { Hello: mockLookupService }, + mockStorage, + mockChainTracker + ) + const steak = await engine.submit({ beef: exampleBeef, topics: ['Hello'] }) + expect(steak).toEqual({ Hello: { outputsToAdmit: [], coinsToRetain: [] } }) + expect(commitAdmission).not.toHaveBeenCalled() + }) + test('does not call commitAdmission and returns the in-memory STEAK when every topic is a dupe', async () => { mockStorage.doesAppliedTransactionExist = jest.fn(async () => true) const onReady = jest.fn() diff --git a/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts b/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts index 7dd05a7ab..e80b3c53c 100644 --- a/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts +++ b/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts @@ -319,6 +319,68 @@ describe('Engine overlay admission helpers', () => { expect(JSON.parse(stale.steak).Hello.coinsRemoved).toEqual([0]) }) + test('carries an explicit coinsRemoved through to a topic STEAK entry that is not re-derived', async () => { + // The top-level STEAK pass (covering every validation, including dupes + // and topics with no new admission) must reflect a caller-supplied + // coinsRemoved rather than always defaulting it to [] -- only the + // *accepted* loop re-derives coinsRemoved from outputsToMarkStale. + const plan = await buildOverlayAdmissionPlan({ + host: host(), + tx: exampleTX, + txid: exampleTxid, + beef: exampleBeef, + topics: ['Hello', 'World'], + mode: 'live', + validations: [ + validation({ isDupe: true, outputsToAdmit: [], coinsRemoved: [3] }), + validation({ topic: 'World' }) + ], + failedTopics: new Set(), + lookupServices, + includePropagation: false + }) + expect(JSON.parse(plan.steak).Hello).toEqual({ + outputsToAdmit: [], + coinsToRetain: [], + coinsRemoved: [3] + }) + }) + + test('recovers a previous txid from sourceTransaction when sourceTXID is absent', async () => { + const provenSource = exampleTX.inputs[0].sourceTransaction + if (provenSource === undefined) throw new Error('expected a proven ancestor') + const tx = { + toBinary: () => exampleTX.toBinary(), + merklePath: undefined, + outputs: exampleTX.outputs, + inputs: [ + { + sourceTXID: undefined, + sourceTransaction: provenSource, + sourceOutputIndex: exampleTX.inputs[0].sourceOutputIndex + } + ] + } as unknown as Transaction + const plan = await buildOverlayAdmissionPlan({ + host: host(), + tx, + txid: exampleTxid, + beef: exampleBeef, + topics: ['Hello'], + mode: 'live', + validations: [validation({ previousCoins: [0], previousOutputs: [previousOutput] })], + failedTopics: new Set(), + lookupServices, + includePropagation: false + }) + expect(plan.decisions[0].evictions).toEqual([ + { + txid: provenSource.id('hex'), + outputIndex: asStorageUint64(String(exampleTX.inputs[0].sourceOutputIndex)) + } + ]) + }) + test('skips previous coins whose source txid cannot be recovered', async () => { const tx = { toBinary: () => exampleTX.toBinary(), From 253391d72ee8c4575e39624ebb79ede763c79b8d Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 20:01:28 -0500 Subject: [PATCH 158/168] test(overlay): cover MongoAdmissionStorage retry/pending orchestration via injected runner New file, mocked MongoTransactionRunner, no real MongoDB: covers isMongoWriteConflict's non-object/null guard, waitForPending's 50-poll timeout falling through to a fresh commit attempt, and non-Mongo/non-Error rejections from the transaction body propagating unclassified rather than being retried or swallowed. Confirms the write-conflict loop's post-loop fallback (`return { state: 'rejected', code: 'spend-conflict' }`) is dead code: the loop's own bound guarantees it always returns/throws by the 8th attempt, so that line can never execute. Co-Authored-By: Claude Fable 5.1 --- ...MongoAdmissionStorageOrchestration.test.ts | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorageOrchestration.test.ts diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorageOrchestration.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorageOrchestration.test.ts new file mode 100644 index 000000000..d5c310053 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorageOrchestration.test.ts @@ -0,0 +1,111 @@ +import type { Db } from 'mongodb' +import type { AdmissionCommit, AdmissionReceipt } from '../../storage/AdmissionStorage.js' +import { admissionReceiptFor } from '../../storage/mongo/MongoAdmissionPlan.js' +import { MongoAdmissionStorage } from '../../storage/mongo/MongoAdmissionStorage.js' +import type { MongoTransactionRunner } from '../../storage/mongo/MongoTransactionRunner.js' +import { admissionPlan } from '../admission/AdmissionStorageContract.js' +import { referenceScope } from '../admission/ReferenceAdmissionStorage.js' + +// These tests exercise MongoAdmissionStorage's own orchestration logic -- +// write-conflict retry exhaustion, the pending-wait poll loop, and its +// classification of thrown errors -- entirely through the injected +// MongoTransactionRunner seam, with no real MongoDB required. Deep +// applyPlan()-internal behavior (the transaction body itself) is covered +// separately against a real replica set in MongoAdmissionStorage.test.ts. + +const dummyDb = { collection: () => ({}) } as unknown as Db + +const clone = (value: T): T => structuredClone(value) + +function plan(): AdmissionCommit { + const next = clone(admissionPlan('orchestration')) + next.decisions[0].reads = [] + return next +} + +function receiptFor(value: AdmissionCommit): AdmissionReceipt { + return admissionReceiptFor(value, []) +} + +function runner( + overrides: Partial> = {} +): MongoTransactionRunner { + return { + reconcile: + overrides.reconcile ?? (async () => ({ state: 'pending' as const, attemptId: 'unlocated' })), + run: + overrides.run ?? + (async () => ({ + state: 'committed' as const, + receipt: receiptFor(plan()) + })), + close: async () => {} + } as unknown as MongoTransactionRunner +} + +function storage(overrides: ConstructorParameters[2] = {}) { + return new MongoAdmissionStorage(dummyDb, referenceScope, { + runner: runner(), + readGuards: { initialize: async () => {} } as never, + ...overrides + }) +} + +describe('MongoAdmissionStorage write-conflict retry exhaustion and pending-wait', () => { + test('exhausting every write-conflict retry attempt surfaces the last raw conflict rather than looping forever', async () => { + const conflict = { code: 112, message: 'transient' } + const run = jest.fn(async () => { + throw conflict + }) + // The retry loop's own bound (attempt < 8, continuing only while + // attempt < 7) guarantees it always returns/throws by the 8th attempt, + // so the last attempt's write conflict -- not classified into any typed + // rejection code by asResult, since it is not an Error instance -- + // propagates as a rejection of commitAdmission itself. + await expect(storage({ runner: runner({ run }) }).commitAdmission(plan())).rejects.toBe( + conflict + ) + expect(run).toHaveBeenCalledTimes(8) + }) + + test('a stuck pending attempt exhausts its 50-poll wait and commitAdmission then submits a fresh attempt', async () => { + const reconcile = jest.fn(async () => ({ + state: 'pending' as const, + attemptId: 'stuck-forever' + })) + const run = jest.fn(async () => ({ state: 'committed' as const, receipt: receiptFor(plan()) })) + // The pre-existing attempt never resolves to committed/rejected/aborted + // across all 50 polls (waitForPending's own timeout, ~1s of 20ms + // delays), so commitAdmission falls through and submits a brand-new + // attempt via the write-conflict retry path instead of hanging forever. + const result = await storage({ runner: runner({ reconcile, run }) }).commitAdmission(plan()) + expect(result).toEqual({ state: 'committed', receipt: receiptFor(plan()) }) + // The initial reconcile at commit entry, plus 50 polls inside waitForPending. + expect(reconcile).toHaveBeenCalledTimes(51) + expect(run).toHaveBeenCalledTimes(1) + }, 10000) + + test('a non-Mongo, non-object rejection from the transaction body is neither retried nor swallowed', async () => { + const run = jest + .fn() + .mockRejectedValueOnce('disk exploded') + .mockResolvedValue({ + state: 'committed' as const, + receipt: receiptFor(plan()) + }) + await expect(storage({ runner: runner({ run }) }).commitAdmission(plan())).rejects.toBe( + 'disk exploded' + ) + // A bare string is not classified as a write conflict, so there is no retry. + expect(run).toHaveBeenCalledTimes(1) + + const runNull = jest.fn(async () => { + // eslint-disable-next-line no-throw-literal + throw null + }) + await expect( + storage({ runner: runner({ run: runNull }) }).commitAdmission(plan()) + ).rejects.toBe(null) + expect(runNull).toHaveBeenCalledTimes(1) + }) +}) From c9af7901365956ed2ab81e3c302130d549d3e201 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 20:01:38 -0500 Subject: [PATCH 159/168] test(overlay): cover MongoAdmissionStorage fail-closed and fencing branches Adds real-replica-set coverage for: a fence conflict rejecting a stale expected history; re-admitting the same tx/topic under a different operation id (fenced off via assertAppliedAvailable, exercising the idempotent-spend return in applySpend along the way); a real eviction succeeding; output/applied-history insert collisions caught as invalid-plan, plus their non-duplicate-key-error fallthroughs (via a namespace-scoped injected write failure); omitting firstSeenHeight; upsertTransaction's raw/manifest presence branches; an unrecognized payload kind rejected while pinning; a payload-store failure while pinning classified as payload-not-ready, and an unrelated one left unclassified; claimOutbox tolerating a since-deleted pinned payload; reconcileAdmission delegating to (and reusing) a scoped peer; applyHandoff's happy path, fence mismatch, and missing-lease paths; and two best-effort concurrent races (generation bootstrap, outbox event id) that each still resolve to exactly one winner and one clean rejection. Co-Authored-By: Claude Fable 5.1 --- .../mongo/MongoAdmissionStorage.test.ts | 458 +++++++++++++++++- 1 file changed, 457 insertions(+), 1 deletion(-) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts index 654390d0e..596745185 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts @@ -3,18 +3,27 @@ import { admissionPlan, admissionStorageContract } from '../admission/AdmissionS import { getAdmissionStorage } from '../../storage/AdmissionStorage.js' import { MongoAdmissionStorage } from '../../storage/mongo/MongoAdmissionStorage.js' import { MongoOverlayStorage } from '../../storage/mongo/MongoOverlayStorage.js' +import { + MongoPayloadStore, + type MongoPayloadReference +} from '../../storage/mongo/MongoPayloadStore.js' +import type { ClientSession } from 'mongodb' import { bootstrapMongoOverlay, encodeMongoOutputIndex, encodeMongoUint64, MongoCollectionNames, + mongoChainKey, mongoNodeKey, mongoRecordKey } from '../../storage/mongo/MongoSchema.js' import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' import { MongoAdmissionHarness } from './MongoAdmissionHarness.js' import type { MongoEnlistedLookupIndex } from '../../storage/mongo/MongoAdmissionStorage.js' -import type { AdmissionCommit } from '../../storage/AdmissionStorage.js' +import type { AdmissionCommit, StorageScope } from '../../storage/AdmissionStorage.js' +import { referenceScope } from '../admission/ReferenceAdmissionStorage.js' + +const clone = (value: T): T => structuredClone(value) describe('Mongo admission storage', () => { let fixture: MongoReplicaFixture @@ -30,6 +39,37 @@ describe('Mongo admission storage', () => { await fixture.close() }, 60000) + /** Seeds everything a plan's decisions and payloads need to be admissible, mirroring the shared contract helper. */ + async function seedPlan( + plan: AdmissionCommit, + options: { skipHistory?: boolean } = {} + ): Promise { + for (const decision of plan.decisions) { + if (!options.skipHistory) { + await harness.seed.history(plan.identity.scope, decision.topic, decision.expectedHistory) + } + for (const read of decision.reads) { + if (read.expectedVersion !== null) { + await harness.seed.read( + plan.identity.scope, + decision.topic, + read.key, + read.expectedVersion + ) + } + } + for (const spend of decision.spends) { + await harness.seed.spendable( + plan.identity.scope, + decision.topic, + spend.outpoint, + spend.expectedVersion + ) + } + } + for (const payload of plan.payloads) await harness.seed.readyPayload(payload) + } + admissionStorageContract(() => harness) test('advertises overlay-admission-v1 only on the complete adapter', () => { @@ -551,6 +591,422 @@ describe('Mongo admission storage', () => { for (const payload of fromZero.payloads) await harness.seed.readyPayload(payload) expect((await harness.adapter.commitAdmission(fromZero)).state).toBe('committed') }) + + test('a decision whose expected history fence no longer matches the topic is rejected as a read conflict', async () => { + await harness.reset() + const plan = admissionPlan('fence-conflict') + // Leave the topic's generation row unseeded so checkHistory sees the + // default zero fence against the plan's own nonzero expectation. + await seedPlan(plan, { skipHistory: true }) + expect(await harness.adapter.commitAdmission(plan)).toEqual({ + state: 'rejected', + code: 'read-conflict' + }) + }) + + test('re-admitting the same tx and topic under a different operation id is fenced off as invalid', async () => { + await harness.reset() + const txid = 'fe'.repeat(32) + const first = admissionPlan('reapply-a', txid) + await seedPlan(first) + expect((await harness.adapter.commitAdmission(first)).state).toBe('committed') + + const second = admissionPlan('reapply-b', txid) + const result = await harness.adapter.commitAdmission(second) + expect(result).toEqual({ state: 'rejected', code: 'invalid-plan' }) + }) + + test('applyEviction transitions a real, currently-unspent output to evicted', async () => { + await harness.reset() + const plan = admissionPlan('real-eviction') + plan.decisions[0].spends = [] + const eviction = { txid: 'ab'.repeat(32), outputIndex: '0' } + plan.decisions[0].evictions = [eviction] + await seedPlan(plan) + await harness.seed.spendable(plan.identity.scope, plan.decisions[0].topic, eviction, '1') + const result = await harness.adapter.commitAdmission(plan) + expect(result.state).toBe('committed') + const raw = await fixture.db + .collection(MongoCollectionNames.outputs) + .findOne({ _id: harness.adapter.outputId(plan.decisions[0].topic, eviction) }) + expect(raw?.state).toBe('evicted') + }) + + test('an output insert that collides with an existing row is rejected as invalid-plan', async () => { + await harness.reset() + const plan = admissionPlan('duplicate-output') + await seedPlan(plan) + const output = plan.decisions[0].outputs[0] + const now = new Date() + // Pre-existing row at the exact _id the plan's own output insert will + // target. Nothing else in applyPlan checks for output existence ahead + // of the insert itself, so this exercises Mongo's own uniqueness guard. + await fixture.db.collection(MongoCollectionNames.outputs).insertOne({ + _id: harness.adapter.outputId(plan.decisions[0].topic, output), + schemaVersion: 1, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash, + nodeId: fixture.scope.nodeId, + topic: plan.decisions[0].topic, + txid: output.txid, + outputIndex: encodeMongoOutputIndex(output.outputIndex), + satoshis: encodeMongoUint64(output.satoshis), + score: encodeMongoUint64(output.score), + scriptPayloadId: 'unrelated-payload', + scriptOffset: encodeMongoUint64('0'), + scriptByteLength: encodeMongoUint64('1'), + state: 'unspent', + version: 'pre-existing', + createdAt: now, + updatedAt: now + }) + const result = await harness.adapter.commitAdmission(plan) + expect(result).toEqual({ state: 'rejected', code: 'invalid-plan' }) + }) + + test('a non-duplicate-key failure inserting an output is rethrown unchanged rather than swallowed', async () => { + await harness.reset() + const plan = admissionPlan('output-insert-failure') + await seedPlan(plan) + // Scope the injected failure to the outputs namespace specifically, so + // it lands on insertOutput's own insertOne rather than the transaction + // runner's unrelated claim-row insert: duplicateKey(error) must be + // false here, so the original error propagates as-is unclassified. + await fixture.failCommands({ + failCommands: ['insert'], + errorCode: 1, + namespace: `${fixture.db.databaseName}.${MongoCollectionNames.outputs}` + }) + await expect(harness.adapter.commitAdmission(plan)).rejects.toMatchObject({ code: 1 }) + await fixture.disableFailPoint() + }) + + test('an applied-history insert that collides under the same admission id is rejected as invalid-plan', async () => { + await harness.reset() + const plan = admissionPlan('duplicate-applied') + await seedPlan(plan) + const appliedId = mongoRecordKey( + mongoNodeKey(plan.identity.scope), + 'applied', + plan.decisions[0].topic, + plan.decisions[0].applied.txid + ) + const now = new Date() + // The admissionId matches this plan's own operationId, so + // assertAppliedAvailable's mismatch guard does not reject it ahead of + // time; only Mongo's own uniqueness guard on the insert itself can. + await fixture.db.collection(MongoCollectionNames.appliedTransactions).insertOne({ + _id: appliedId, + schemaVersion: 1, + network: plan.identity.scope.network, + genesisHash: plan.identity.scope.genesisHash, + nodeId: plan.identity.scope.nodeId, + topic: plan.decisions[0].topic, + txid: plan.decisions[0].applied.txid, + state: 'unproven', + admissionId: plan.key.operationId, + createdAt: now, + updatedAt: now + }) + const result = await harness.adapter.commitAdmission(plan) + expect(result).toEqual({ state: 'rejected', code: 'invalid-plan' }) + }) + + test('a non-duplicate-key failure inserting an applied-history row is rethrown unchanged rather than swallowed', async () => { + await harness.reset() + const plan = admissionPlan('applied-insert-failure') + await seedPlan(plan) + // Scope the injected failure to the appliedTransactions namespace so it + // lands on insertApplied's own insertOne (which runs after the output + // has already been inserted successfully), not an earlier, unrelated + // insert -- and duplicateKey(error) must be false here, so the original + // error propagates as-is instead of being reclassified. + await fixture.failCommands({ + failCommands: ['insert'], + errorCode: 1, + namespace: `${fixture.db.databaseName}.${MongoCollectionNames.appliedTransactions}` + }) + await expect(harness.adapter.commitAdmission(plan)).rejects.toMatchObject({ code: 1 }) + await fixture.disableFailPoint() + }) + + test('omits firstSeenHeight from the applied-history row when the plan does not supply one', async () => { + await harness.reset() + const plan = clone(admissionPlan('no-first-seen')) + delete (plan.decisions[0].applied as { firstSeenHeight?: string }).firstSeenHeight + await seedPlan(plan) + expect((await harness.adapter.commitAdmission(plan)).state).toBe('committed') + const appliedId = mongoRecordKey( + mongoNodeKey(plan.identity.scope), + 'applied', + plan.decisions[0].topic, + plan.decisions[0].applied.txid + ) + const raw = await fixture.db + .collection(MongoCollectionNames.appliedTransactions) + .findOne({ _id: appliedId }) + expect(raw?.firstSeenHeight).toBeUndefined() + }) + + test('upsertTransaction omits rawPayloadId and records manifestPayloadId when the plan carries no raw payload', async () => { + await harness.reset() + const plan = clone(admissionPlan('no-raw-with-manifest')) + plan.outbox = [] + plan.payloads = plan.payloads.filter(ref => ref.kind !== 'raw-transaction') + plan.payloads.push({ kind: 'beef-manifest', digest: 'ef'.repeat(32), byteLength: '12' }) + await seedPlan(plan) + expect((await harness.adapter.commitAdmission(plan)).state).toBe('committed') + const id = mongoRecordKey(mongoChainKey(plan.identity.scope), 'transaction', plan.identity.txid) + const raw = await fixture.db.collection(MongoCollectionNames.transactions).findOne({ _id: id }) + expect(raw?.rawPayloadId).toBeUndefined() + expect(raw?.manifestPayloadId).toBe( + harness.adapter.payloadId({ kind: 'beef-manifest', digest: 'ef'.repeat(32) }) + ) + }) + + test('an output payload ref with an unrecognized kind is rejected even though it is otherwise well-formed', async () => { + await harness.reset() + const plan = clone(admissionPlan('weird-payload-kind')) + const weirdRef = { kind: 'weird-kind', digest: 'cd'.repeat(32), byteLength: '2' } + ;( + plan.decisions[0].outputs[0].script as unknown as { + payload: { kind: string; digest: string; byteLength: string } + } + ).payload = weirdRef + await seedPlan(plan) + await harness.seed.readyPayload( + weirdRef as unknown as Parameters[0] + ) + const result = await harness.adapter.commitAdmission(plan) + expect(result).toEqual({ state: 'rejected', code: 'invalid-plan' }) + }) + + test('a payload store failure while pinning an output script surfaces as payload-not-ready', async () => { + await harness.reset() + // admissionPlan() binds its identity/key to referenceScope, not the + // fixture's own randomly-named scope, so the adapter under test must be + // constructed with the same scope -- otherwise commitAdmission's own + // cross-scope guard would silently delegate to an unwrapped peer. + const realPayloads = new MongoPayloadStore(fixture.db, referenceScope) + const payloads = { + publish: realPayloads.publish.bind(realPayloads), + addReference: async ( + session: ClientSession, + reference: MongoPayloadReference, + operation?: { timeoutMS?: number; signal?: AbortSignal } + ) => { + if (reference.ownerKind === 'output' && reference.slot === 'script') { + throw new Error('Mongo payload is not ready for reference') + } + await realPayloads.addReference(session, reference, operation) + }, + releaseReference: realPayloads.releaseReference.bind(realPayloads) + } as unknown as MongoPayloadStore + const adapter = new MongoAdmissionStorage(fixture.db, referenceScope, { payloads }) + const plan = admissionPlan('pin-store-failure') + await seedPlan(plan) + const result = await adapter.commitAdmission(plan) + expect(result).toEqual({ state: 'rejected', code: 'payload-not-ready' }) + await adapter.close() + }) + + test('an unrelated payload store failure while pinning propagates unchanged rather than being reclassified', async () => { + await harness.reset() + const realPayloads = new MongoPayloadStore(fixture.db, referenceScope) + const payloads = { + publish: realPayloads.publish.bind(realPayloads), + addReference: async ( + session: ClientSession, + reference: MongoPayloadReference, + operation?: { timeoutMS?: number; signal?: AbortSignal } + ) => { + if (reference.ownerKind === 'output' && reference.slot === 'script') { + throw new Error('Mongo payload store is temporarily unavailable') + } + await realPayloads.addReference(session, reference, operation) + }, + releaseReference: realPayloads.releaseReference.bind(realPayloads) + } as unknown as MongoPayloadStore + const adapter = new MongoAdmissionStorage(fixture.db, referenceScope, { payloads }) + const plan = admissionPlan('pin-unrelated-failure') + await seedPlan(plan) + await expect(adapter.commitAdmission(plan)).rejects.toThrow( + 'Mongo payload store is temporarily unavailable' + ) + await adapter.close() + }) + + test('claimOutbox omits a pinned payload whose underlying document has since been removed', async () => { + await harness.reset() + const plan = admissionPlan('outbox-missing-payload') + await seedPlan(plan) + expect((await harness.adapter.commitAdmission(plan)).state).toBe('committed') + const outboxDataRef = plan.payloads.find(ref => ref.kind === 'outbox-data') + if (outboxDataRef === undefined) throw new Error('expected an outbox-data payload in the plan') + await fixture.db + .collection(MongoCollectionNames.payloads) + .deleteOne({ _id: harness.adapter.payloadId(outboxDataRef) }) + const claimed = await harness.adapter.claimOutbox('lookup') + expect(claimed?.eventId).toBe(`${plan.key.operationId}:lookup`) + expect(claimed?.payloads).toEqual([]) + }) + + test('reconcileAdmission for a different scope delegates to, and reuses, a scoped peer instance', async () => { + await harness.reset() + const otherScope: StorageScope = { ...fixture.scope, nodeId: `${fixture.scope.nodeId}-peer` } + const key = { + scope: otherScope, + operationId: 'never-committed-on-peer-scope', + semanticDigest: 'ab'.repeat(32) + } + // A never-committed operation reconciles to a fresh "pending/unlocated" + // result from the peer's own MongoTransactionRunner; a second call for + // the same foreign scope must reuse the cached peer rather than + // constructing (and reconnecting) a fresh one every time. + const first = await harness.adapter.reconcileAdmission(key) + const second = await harness.adapter.reconcileAdmission(key) + expect(first).toEqual({ state: 'pending', attemptId: 'unlocated' }) + expect(second).toEqual(first) + }) + + test('applyHandoff advances a matching, unexpired lease and records its checkpoint', async () => { + await harness.reset() + const plan = admissionPlan('handoff-happy') + plan.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '4', + affectedFromHeight: '99', + handoff: { + expected: { + scope: plan.identity.scope, + topic: plan.decisions[0].topic, + peerId: 'peer-a', + jobId: 'job-a', + leaseToken: '9', + expiresAtMs: '1', + chainEpoch: '7', + topicHistoryGeneration: '3' + }, + checkpoint: 'checkpoint-a' + } + } + await seedPlan(plan) + await harness.seed.lease(plan.decisions[0].historyUpdate.handoff.expected) + expect((await harness.adapter.commitAdmission(plan)).state).toBe('committed') + const jobId = mongoRecordKey( + mongoNodeKey(plan.identity.scope), + 'job', + plan.decisions[0].topic, + 'peer-a', + 'job-a' + ) + const raw = await fixture.db + .collection(MongoCollectionNames.basmRecoveryJobs) + .findOne({ _id: jobId }) + expect(raw?.checkpoint).toBe('checkpoint-a') + expect(raw?.topicHistoryGeneration).toBe(encodeMongoUint64('4')) + }) + + test('applyHandoff rejects a handoff whose expected fence does not match the decision', async () => { + await harness.reset() + const plan = admissionPlan('handoff-mismatch') + plan.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '4', + affectedFromHeight: '99', + handoff: { + expected: { + scope: plan.identity.scope, + topic: plan.decisions[0].topic, + peerId: 'peer-b', + jobId: 'job-b', + leaseToken: '9', + expiresAtMs: '1', + chainEpoch: '1', + topicHistoryGeneration: '3' + }, + checkpoint: 'checkpoint-b' + } + } + await seedPlan(plan) + await harness.seed.lease(plan.decisions[0].historyUpdate.handoff.expected) + expect(await harness.adapter.commitAdmission(plan)).toEqual({ + state: 'rejected', + code: 'read-conflict' + }) + }) + + test('applyHandoff rejects when no matching, unexpired lease exists for the topic revision handoff', async () => { + await harness.reset() + const plan = admissionPlan('handoff-missing-job') + plan.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '4', + affectedFromHeight: '99', + handoff: { + expected: { + scope: plan.identity.scope, + topic: plan.decisions[0].topic, + peerId: 'peer-c', + jobId: 'job-c', + leaseToken: '9', + expiresAtMs: '1', + chainEpoch: '7', + topicHistoryGeneration: '3' + }, + checkpoint: 'checkpoint-c' + } + } + await seedPlan(plan) + // No lease is seeded for job-c: applyHandoff's own findOneAndUpdate must + // fail closed rather than silently skipping the checkpoint update. + expect(await harness.adapter.commitAdmission(plan)).toEqual({ + state: 'rejected', + code: 'read-conflict' + }) + }) + + test('two admissions racing to bootstrap the same topic generation resolve to exactly one winner', async () => { + await harness.reset() + const bootstrapPlan = (operationId: string, txid: string): AdmissionCommit => { + const value = admissionPlan(operationId, txid) + value.decisions[0].expectedHistory = { chainEpoch: '0', topicHistoryGeneration: '0' } + value.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '1', + affectedFromHeight: '0' + } + value.decisions[0].spends = [] + return value + } + const a = bootstrapPlan('bootstrap-race-a', 'aa'.repeat(32)) + const b = bootstrapPlan('bootstrap-race-b', 'bb'.repeat(32)) + for (const value of [a, b]) await seedPlan(value) + const results = await Promise.all([ + harness.adapter.commitAdmission(a), + harness.adapter.commitAdmission(b) + ]) + expect(results.filter(result => result.state === 'committed')).toHaveLength(1) + expect( + results.filter(result => result.state === 'rejected' && result.code === 'read-conflict') + ).toHaveLength(1) + }) + + test('two admissions racing on the same outbox event id resolve to exactly one winner', async () => { + await harness.reset() + const sharedEventId = 'shared-outbox-event' + const racePlan = (operationId: string, txid: string): AdmissionCommit => { + const value = admissionPlan(operationId, txid) + value.decisions[0].spends = [] + value.outbox = [{ ...value.outbox[0], eventId: sharedEventId }] + return value + } + const a = racePlan('outbox-race-a', 'cc'.repeat(32)) + const b = racePlan('outbox-race-b', 'dd'.repeat(32)) + for (const value of [a, b]) await seedPlan(value) + const results = await Promise.all([ + harness.adapter.commitAdmission(a), + harness.adapter.commitAdmission(b) + ]) + expect(results.filter(result => result.state === 'committed')).toHaveLength(1) + expect(results.filter(result => result.state === 'rejected')).toHaveLength(1) + }) }) function hydrationBeef(): number[] { From 98b4b75e86223d86b9ba06fe3b550e88c0784483 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 20:01:47 -0500 Subject: [PATCH 160/168] test(overlay): cover MongoOverlayStorage read/hydration branches Covers: reusing a caller-supplied payload store instead of constructing one; insertOutput recording a spent output from the start; findOutput with includeBEEF leaving beef unset when no transaction row exists, and when the transaction row names a payload that is no longer ready; readScript returning an empty script for a (schema-legacy, simulated via bypassDocumentValidation) output document with no scriptPayloadId; and readBeef hydrating a real merkle path pinned by an admission commit into an atomic BEEF, then falling back to the plain BEEF when that merkle payload is later removed. Co-Authored-By: Claude Fable 5.1 --- .../mongo/MongoOverlayStorageEviction.test.ts | 233 +++++++++++++++++- 1 file changed, 228 insertions(+), 5 deletions(-) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageEviction.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageEviction.test.ts index 5ed1c1f39..0e2179aeb 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageEviction.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageEviction.test.ts @@ -1,8 +1,19 @@ import { randomUUID } from 'node:crypto' +import { Transaction } from '@bsv/sdk' import type { Output } from '../../Output.js' import { MongoOverlayStorage } from '../../storage/mongo/MongoOverlayStorage.js' -import { bootstrapMongoOverlay, MongoCollectionNames } from '../../storage/mongo/MongoSchema.js' +import { MongoPayloadStore } from '../../storage/mongo/MongoPayloadStore.js' +import { + bootstrapMongoOverlay, + encodeMongoOutputIndex, + encodeMongoUint64, + MongoCollectionNames +} from '../../storage/mongo/MongoSchema.js' import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' +import { admissionSemanticDigest, type AdmissionCommit } from '../../storage/AdmissionStorage.js' + +const BRC62Hex = + '0100beef01fe636d0c0007021400fe507c0c7aa754cef1f7889d5fd395cf1f785dd7de98eed895dbedfe4e5bc70d1502ac4e164f5bc16746bb0868404292ac8318bbac3800e4aad13a014da427adce3e010b00bc4ff395efd11719b277694cface5aa50d085a0bb81f613f70313acd28cf4557010400574b2d9142b8d28b61d88e3b2c3f44d858411356b49a28a4643b6d1a6a092a5201030051a05fc84d531b5d250c23f4f886f6812f9fe3f402d61607f977b4ecd2701c19010000fd781529d58fc2523cf396a7f25440b409857e7e221766c57214b1d38c7b481f01010062f542f45ea3660f86c013ced80534cb5fd4c19d66c56e7e8c5d4bf2d40acc5e010100b121e91836fd7cd5102b654e9f72f3cf6fdbfd0b161c53a9c54b12c841126331020100000001cd4e4cac3c7b56920d1e7655e7e260d31f29d9a388d04910f1bbd72304a79029010000006b483045022100e75279a205a547c445719420aa3138bf14743e3f42618e5f86a19bde14bb95f7022064777d34776b05d816daf1699493fcdf2ef5a5ab1ad710d9c97bfb5b8f7cef3641210263e2dee22b1ddc5e11f6fab8bcd2378bdd19580d640501ea956ec0e786f93e76ffffffff013e660000000000001976a9146bfd5c7fbe21529d45803dbcf0c87dd3c71efbc288ac0000000001000100000001ac4e164f5bc16746bb0868404292ac8318bbac3800e4aad13a014da427adce3e000000006a47304402203a61a2e931612b4bda08d541cfb980885173b8dcf64a3471238ae7abcd368d6402204cbf24f04b9aa2256d8901f0ed97866603d2be8324c2bfb7a37bf8fc90edd5b441210263e2dee22b1ddc5e11f6fab8bcd2378bdd19580d640501ea956ec0e786f93e76ffffffff013c660000000000001976a9146bfd5c7fbe21529d45803dbcf0c87dd3c71efbc288ac0000000000' // Direct access to the private mapping function this finding names: it is // the single place responsible for never surfacing a non-serving (evicted) @@ -18,7 +29,11 @@ type ToOutputHost = { } function randomHex(bytes: number): string { - return Array.from({ length: bytes }, () => Math.floor(Math.random() * 256).toString(16).padStart(2, '0')).join('') + return Array.from({ length: bytes }, () => + Math.floor(Math.random() * 256) + .toString(16) + .padStart(2, '0') + ).join('') } describe('MongoOverlayStorage eviction and consumption-edge hydration', () => { @@ -51,9 +66,12 @@ describe('MongoOverlayStorage eviction and consumption-edge hydration', () => { await storage.insertOutput(output) await storage.deleteOutput(txid, 0, topic) - const raw = await fixture.db - .collection(MongoCollectionNames.outputs) - .findOne({ txid, topic, network: fixture.scope.network, genesisHash: fixture.scope.genesisHash }) + const raw = await fixture.db.collection(MongoCollectionNames.outputs).findOne({ + txid, + topic, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash + }) expect(raw?.state).toBe('evicted') if (raw === null) throw new Error('expected the evicted document to still exist for audit') @@ -106,4 +124,209 @@ describe('MongoOverlayStorage eviction and consumption-edge hydration', () => { expect(forTransaction).toHaveLength(1) expect(forTransaction[0].outputsConsumed).toEqual([{ txid: sourceTxid, outputIndex: 0 }]) }) + + test('uses a caller-supplied payload store instead of constructing its own', async () => { + const real = new MongoPayloadStore(fixture.db, fixture.scope) + const calls: string[] = [] + const payloads = { + publish: async (input: Parameters[0]) => { + calls.push(input.kind) + return await real.publish(input) + } + } as unknown as MongoPayloadStore + const overlay = new MongoOverlayStorage(fixture.db, fixture.scope, { payloads }) + await overlay.publishAdmissionPayload({ kind: 'locking-script', bytes: new Uint8Array([7]) }) + // Proves the constructor actually used the injected store (the + // `options.payloads ?? new MongoPayloadStore(...)` branch) rather than + // silently falling back to one it constructs itself. + expect(calls).toEqual(['locking-script']) + await overlay.close() + }) + + test('insertOutput records a spent output as spent from the start', async () => { + const txid = randomHex(32) + const topic = `Spent-${randomUUID()}` + await storage.insertOutput({ + txid, + outputIndex: 0, + outputScript: [4, 5], + satoshis: 10, + topic, + spent: true, + outputsConsumed: [], + consumedBy: [] + }) + expect(await storage.findOutput(txid, 0, topic, true)).toEqual( + expect.objectContaining({ spent: true }) + ) + expect(await storage.findOutput(txid, 0, topic, false)).toBeNull() + }) + + test('findOutput with includeBEEF leaves beef unset when no transaction row exists for the output', async () => { + const txid = randomHex(32) + const topic = `NoBeef-${randomUUID()}` + // insertOutput without a `beef` field never calls persistTransactionBeef, + // so no row exists in the transactions collection for this txid at all. + await storage.insertOutput({ + txid, + outputIndex: 0, + outputScript: [9, 9], + satoshis: 1, + topic, + spent: false, + outputsConsumed: [], + consumedBy: [] + }) + const found = await storage.findOutput(txid, 0, topic, false, true) + expect(found).not.toBeNull() + expect(found?.beef).toBeUndefined() + }) + + test('readScript returns an empty script when the output document carries no scriptPayloadId', async () => { + const txid = randomHex(32) + const topic = `NoScript-${randomUUID()}` + const now = new Date() + // The live schema validator requires scriptPayloadId on every write, so + // a document missing it can only arrive here as a pre-existing (e.g. + // migrated, or written under an older schema) row -- simulated with + // bypassDocumentValidation rather than a normal insertOutput() + $unset, + // which the validator would reject outright. + await fixture.db.collection(MongoCollectionNames.outputs).insertOne( + { + _id: `no-script-${randomUUID()}`, + schemaVersion: 1, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash, + nodeId: fixture.scope.nodeId, + topic, + txid, + outputIndex: encodeMongoOutputIndex('0'), + satoshis: encodeMongoUint64('1'), + score: encodeMongoUint64('1'), + scriptOffset: encodeMongoUint64('0'), + scriptByteLength: encodeMongoUint64('0'), + state: 'unspent', + version: '1', + createdAt: now, + updatedAt: now + }, + { bypassDocumentValidation: true } + ) + const found = await storage.findOutput(txid, 0, topic) + expect(found?.outputScript).toEqual([]) + }) + + test('findOutput with includeBEEF leaves beef unset when the transaction row names a payload that is no longer ready', async () => { + const tx = Transaction.fromHexBEEF(BRC62Hex) + const txid = tx.id('hex') + const beef = tx.toBEEF() + const topic = `EvictedRaw-${randomUUID()}` + await storage.insertOutput({ + txid, + outputIndex: 0, + outputScript: [1, 1], + satoshis: 1, + topic, + spent: false, + outputsConsumed: [], + consumedBy: [], + beef + }) + // Simulate a raw-transaction payload that has since been garbage + // collected (or is otherwise no longer ready) while the transaction row + // that named it is left in place -- readBeef must fail closed to + // "no BEEF" rather than surfacing a null-pointer or stale bytes. + const transaction = await fixture.db + .collection(MongoCollectionNames.transactions) + .findOne({ txid }) + expect(typeof transaction?.rawPayloadId).toBe('string') + await fixture.db + .collection(MongoCollectionNames.payloads) + .updateOne({ _id: transaction?.rawPayloadId as string }, { $set: { state: 'deleted' } }) + const found = await storage.findOutput(txid, 0, topic, false, true) + expect(found?.beef).toBeUndefined() + }) + + test('readBeef hydrates a merkle path pinned by an admission commit into the returned atomic BEEF', async () => { + const exampleTX = Transaction.fromHexBEEF(BRC62Hex) + const proven = exampleTX.inputs[0].sourceTransaction + if (proven === undefined || proven.merklePath === undefined) { + throw new Error('expected the BEEF ancestor to carry a merkle path') + } + const txid = proven.id('hex') + const topic = `Merkle-${randomUUID()}` + const raw = await storage.publishAdmissionPayload({ + kind: 'raw-transaction', + bytes: Buffer.from(proven.toBinary()), + txid + }) + const merkle = await storage.publishAdmissionPayload({ + kind: 'merkle-path', + bytes: Buffer.from(proven.merklePath.toBinary()) + }) + const script = await storage.publishAdmissionPayload({ + kind: 'locking-script', + bytes: Buffer.from(proven.outputs[0].lockingScript.toBinary()) + }) + const identity = { + scope: storage.admissionScope, + txid, + mode: 'historical' as const, + contextDigest: '00'.repeat(32), + topics: [{ topic, policyId: 'merkle-hydration-test' }] + } + const plan: AdmissionCommit = { + key: { + scope: storage.admissionScope, + operationId: `merkle-hydrate-${txid}`, + semanticDigest: admissionSemanticDigest(identity) + }, + identity, + payloads: [raw], + decisions: [ + { + topic, + expectedHistory: { chainEpoch: '0', topicHistoryGeneration: '0' }, + reads: [], + spends: [], + evictions: [], + outputs: [ + { + txid, + outputIndex: '0', + satoshis: String(proven.outputs[0].satoshis), + score: '1', + script: { payload: script, offset: '0', byteLength: script.byteLength } + } + ], + edges: [], + applied: { txid, proof: merkle, firstSeenHeight: '1' } + } + ], + outbox: [], + steak: JSON.stringify({ + [topic]: { outputsToAdmit: [0], coinsToRetain: [], coinsRemoved: [] } + }) + } + const committed = await storage.admission.commitAdmission(plan) + expect(committed.state).toBe('committed') + const found = await storage.findOutput(txid, 0, topic, false, true) + expect(found?.beef).toBeDefined() + const rebuilt = Transaction.fromBEEF(found?.beef ?? []) + expect(rebuilt.id('hex')).toBe(txid) + expect(rebuilt.merklePath).toBeDefined() + + // The merkle-path *reference* still names this transaction, but the + // payload it points at is no longer ready (GC'd, or never finished + // uploading) -- readBeef must fall back to the plain (non-atomic) BEEF + // rather than crashing or fabricating a merkle path. + await fixture.db + .collection(MongoCollectionNames.payloads) + .updateOne({ _id: storage.admission.payloadId(merkle) }, { $set: { state: 'deleted' } }) + const withoutMerkle = await storage.findOutput(txid, 0, topic, false, true) + expect(withoutMerkle?.beef).toBeDefined() + const rebuiltWithoutMerkle = Transaction.fromBEEF(withoutMerkle?.beef ?? []) + expect(rebuiltWithoutMerkle.id('hex')).toBe(txid) + expect(rebuiltWithoutMerkle.merklePath).toBeUndefined() + }) }) From aac4925c9a53358fb54bee96e31f0da6128880a9 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 20:01:56 -0500 Subject: [PATCH 161/168] test(overlay): cover MongoPayloadStore read()/readBytes() in a new file New file, scoped to read()/readBytes() only per this PR's worklist (a concurrent branch owns the rest of MongoPayloadStore.ts's coverage, in its own MongoPayloadStore.test.ts, so this stays a separate file to keep the eventual merge conflict-free). Covers: read() on missing content; read() rejecting a negative offset, a negative length, and an offset+length past the end; read() rejecting a range whose end would exceed a safe JS integer; readBytes() downloading GridFS-spilled content across chunks (including a partial, mid-file range); and readBytes() failing closed on a missing GridFS file and on a ready record with neither inline data nor a file id. Co-Authored-By: Claude Fable 5.1 --- .../mongo/MongoPayloadStoreRead.test.ts | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoPayloadStoreRead.test.ts diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStoreRead.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStoreRead.test.ts new file mode 100644 index 000000000..b54fbf3c9 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStoreRead.test.ts @@ -0,0 +1,193 @@ +import { createHash, randomUUID } from 'node:crypto' +import { ObjectId } from 'mongodb' +import { + bootstrapMongoOverlay, + encodeMongoUint64, + mongoChainKey, + mongoRecordKey, + MongoCollectionNames +} from '../../storage/mongo/MongoSchema.js' +import { MongoPayloadStore } from '../../storage/mongo/MongoPayloadStore.js' +import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' + +// Scoped to MongoPayloadStore.read()/readBytes() specifically (per the +// worklist for this PR). Every other MongoPayloadStore.ts line belongs to a +// different PR's worklist and is covered in MongoPayloadStore.test.ts on its +// own branch; this file stays separate so the two never collide on a merge. + +const bytes = async function* ( + value: Uint8Array, + split = value.byteLength +): AsyncIterable { + for (let index = 0; index < value.byteLength; index += split) + yield value.subarray(index, index + split) +} + +const digest = (value: Uint8Array): string => createHash('sha256').update(value).digest('hex') + +describe('MongoPayloadStore.read() and readBytes()', () => { + let fixture: MongoReplicaFixture + let store: MongoPayloadStore + + beforeAll(async () => { + fixture = await createMongoReplicaFixture() + await bootstrapMongoOverlay(fixture.db, fixture.scope) + store = new MongoPayloadStore(fixture.db, fixture.scope) + }, 120000) + + afterAll(async () => { + await fixture.close() + }, 30000) + + test('read() throws when no ready payload exists for the requested content', async () => { + await expect(store.read({ kind: 'locking-script', digest: 'aa'.repeat(32) })).rejects.toThrow( + 'Mongo payload is not ready' + ) + }) + + test('read() rejects a range whose offset or length is negative, or whose end exceeds the content', async () => { + const content = Buffer.from('range-guard-content') + const hash = digest(content) + await store.publish({ + kind: 'locking-script', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const ref = { kind: 'locking-script' as const, digest: hash } + + await expect( + store.read(ref, { offset: '-1', byteLength: String(content.byteLength) }) + ).rejects.toThrow('Mongo payload range is invalid') + await expect(store.read(ref, { offset: '0', byteLength: '-1' })).rejects.toThrow( + 'Mongo payload range is invalid' + ) + await expect( + store.read(ref, { offset: '1', byteLength: String(content.byteLength) }) + ).rejects.toThrow('Mongo payload range is invalid') + + // A range that fits exactly still succeeds, proving the guard is not + // simply rejecting every explicit range. + const whole = await store.read(ref, { offset: '0', byteLength: String(content.byteLength) }) + expect(Buffer.from(whole).toString()).toBe(content.toString()) + }) + + test('read() rejects a range whose end would exceed a safe JavaScript integer', async () => { + // Directly seed a "ready" payload record whose declared byteLength is + // enormous (but a legal uint64), without ever streaming that much real + // content -- read() must fail closed on the arithmetic before it ever + // attempts to fetch bytes. + const hugeLength = '18446744073709551615' + const id = mongoRecordKey(mongoChainKey(fixture.scope), 'locking-script', 'bb'.repeat(32)) + const now = new Date() + await fixture.db.collection(MongoCollectionNames.payloads).insertOne({ + _id: id, + schemaVersion: 1, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash, + kind: 'locking-script', + digest: 'bb'.repeat(32), + byteLength: encodeMongoUint64(hugeLength), + state: 'ready', + guard: randomUUID(), + ownerNodeId: fixture.scope.nodeId, + ownerId: randomUUID(), + fencingToken: encodeMongoUint64('1'), + leaseUntil: now, + createdAt: now, + updatedAt: now + }) + await expect( + store.read( + { kind: 'locking-script', digest: 'bb'.repeat(32) }, + { offset: '0', byteLength: hugeLength } + ) + ).rejects.toThrow('Mongo payload range exceeds safe integer') + }) + + test('readBytes() downloads content that spilled to GridFS across several chunks', async () => { + const tight = new MongoPayloadStore(fixture.db, fixture.scope, { + inlineCeilingBytes: 16, + gridFsChunkBytes: 8 * 1024 + }) + const content = Buffer.from( + 'this content is well over the sixteen byte inline ceiling for this test' + ) + const hash = digest(content) + const published = await tight.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content, 10) + }) + const row = await fixture.db + .collection(MongoCollectionNames.payloads) + .findOne({ digest: hash, kind: 'outbox-data' }) + expect(row?.fileId).toBeDefined() + expect(row?.inlineData).toBeUndefined() + + const roundTripped = await tight.read(published) + expect(Buffer.from(roundTripped).toString()).toBe(content.toString()) + + // A partial, mid-file range read must still line up byte-for-byte. + const slice = await tight.read(published, { offset: '5', byteLength: '10' }) + expect(Buffer.from(slice).toString()).toBe(content.subarray(5, 15).toString()) + }) + + test('readBytes() surfaces a download failure when the GridFS file behind a ready record is gone', async () => { + const now = new Date() + const missingFileId = new ObjectId() + const id = mongoRecordKey(mongoChainKey(fixture.scope), 'outbox-data', 'cc'.repeat(32)) + await fixture.db.collection(MongoCollectionNames.payloads).insertOne({ + _id: id, + schemaVersion: 1, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash, + kind: 'outbox-data', + digest: 'cc'.repeat(32), + byteLength: encodeMongoUint64('4'), + state: 'ready', + guard: randomUUID(), + ownerNodeId: fixture.scope.nodeId, + ownerId: randomUUID(), + fencingToken: encodeMongoUint64('1'), + leaseUntil: now, + createdAt: now, + updatedAt: now, + // Points at a GridFS file that was never uploaded (deleted/evicted), + // simulating a ready record left behind after its content was reclaimed. + fileId: missingFileId + }) + await expect(store.read({ kind: 'outbox-data', digest: 'cc'.repeat(32) })).rejects.toThrow( + /FileNotFound/ + ) + }) + + test('readBytes() fails closed when a ready record carries neither inline data nor a GridFS file', async () => { + const now = new Date() + const id = mongoRecordKey(mongoChainKey(fixture.scope), 'outbox-data', 'dd'.repeat(32)) + // A record can only reach `ready` through completeUpload(), which always + // sets exactly one of inlineData/fileId -- this simulates a corrupted or + // hand-edited row that skipped that invariant. + await fixture.db.collection(MongoCollectionNames.payloads).insertOne({ + _id: id, + schemaVersion: 1, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash, + kind: 'outbox-data', + digest: 'dd'.repeat(32), + byteLength: encodeMongoUint64('4'), + state: 'ready', + guard: randomUUID(), + ownerNodeId: fixture.scope.nodeId, + ownerId: randomUUID(), + fencingToken: encodeMongoUint64('1'), + leaseUntil: now, + createdAt: now, + updatedAt: now + }) + await expect(store.read({ kind: 'outbox-data', digest: 'dd'.repeat(32) })).rejects.toThrow( + 'Mongo ready payload has no bytes' + ) + }) +}) From 3ca6cd20d80d8f491efed2ff6be279123e8f2590 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 20:18:16 -0500 Subject: [PATCH 162/168] test(overlay): fix scope mismatch and range invariant, add GC-tolerant pin coverage Two follow-ups to the prior MongoAdmissionStorage commit: the unrecognized-payload-kind test kept the output's script offset/byteLength pointed at the original (25-byte) script, so isInvalidOutput's own range check rejected the plan before pin() was ever reached -- fixed to match the substituted payload's declared byteLength, which now actually reaches and exercises pin()'s isPayloadKind guard. Also adds coverage for replaceHistoryPin tolerating a previous history-update pin whose payload document has since been garbage collected while its reference row remains, skipping the release instead of dereferencing a missing document. Co-Authored-By: Claude Fable 5.1 --- .../mongo/MongoAdmissionStorage.test.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts index 596745185..e4430e8ca 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoAdmissionStorage.test.ts @@ -314,6 +314,40 @@ describe('Mongo admission storage', () => { expect(snapshot.historyUpdates).toEqual([{ topic: 'tm_contract', affectedFromHeight: '100' }]) }) + test('replaceHistoryPin tolerates a previous pin whose payload document is already gone', async () => { + await harness.reset() + const first = admissionPlan('history-pin-gc-first') + first.decisions[0].historyUpdate = { nextTopicHistoryGeneration: '4', affectedFromHeight: '99' } + await seedPlan(first) + expect((await harness.adapter.commitAdmission(first)).state).toBe('committed') + + // Simulate the previous history-update payload having since been + // garbage collected while its pin reference row is still in place: + // replaceHistoryPin must skip the release rather than crash on it. + const reference = await fixture.db.collection(MongoCollectionNames.payloadReferences).findOne({ + network: first.identity.scope.network, + genesisHash: first.identity.scope.genesisHash, + nodeId: first.identity.scope.nodeId, + ownerKind: 'basm-job', + ownerId: 'tm_contract', + slot: 'history-update:4' + }) + expect(typeof reference?.payloadId).toBe('string') + await fixture.db + .collection(MongoCollectionNames.payloads) + .deleteOne({ _id: reference?.payloadId as string }) + + const second = admissionPlan('history-pin-gc-second', '6'.repeat(64)) + second.decisions[0].spends = [] + second.decisions[0].expectedHistory = { chainEpoch: '7', topicHistoryGeneration: '4' } + second.decisions[0].historyUpdate = { + nextTopicHistoryGeneration: '5', + affectedFromHeight: '100' + } + await seedPlan(second) + expect((await harness.adapter.commitAdmission(second)).state).toBe('committed') + }) + test('hydrates output scripts and BEEF from payload bytes', async () => { await harness.reset() const storage = new MongoOverlayStorage(fixture.db, fixture.scope) @@ -768,6 +802,12 @@ describe('Mongo admission storage', () => { await harness.reset() const plan = clone(admissionPlan('weird-payload-kind')) const weirdRef = { kind: 'weird-kind', digest: 'cd'.repeat(32), byteLength: '2' } + // The script's own offset/byteLength must stay consistent with the + // (weird-kind) payload's declared byteLength, or isInvalidOutput's + // range check rejects the plan for a different reason before applyPlan + // -- and pin() -- is ever reached. + plan.decisions[0].outputs[0].script.offset = '0' + plan.decisions[0].outputs[0].script.byteLength = weirdRef.byteLength ;( plan.decisions[0].outputs[0].script as unknown as { payload: { kind: string; digest: string; byteLength: string } From acee5adb9fccfeb9331b19f64d64959b2df96acc Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 20:20:07 -0500 Subject: [PATCH 163/168] test(overlay): pin MongoPayloadStore fail-closed and fencing behavior Adds real jest tests for previously-uncovered MongoPayloadStore branches: pin re-add no-op, claimGarbage/finishGarbage edge cases (never-published digest, afterDeleteClaim hook, ownership-metadata drift), lease-stolen fencing before the ready CAS and before GridFS staging, wrong explicit txid on both the fresh-upload and already-ready paths, GridFS crash-boundary hooks, abandonUpload's guard against deleting a concurrently-published file, digest-mismatch-after-spillover cleanup, stale inline-upload recovery, an expired crashed reservation reclaiming and retiring its orphaned GridFS file, and the abort-signal fallback messages used when a signal's `reason` is unset. Also stresses a many-way concurrent first-time publish of the same digest and a corrupted-in-place staged GridFS chunk to exercise the duplicate-key race and post-upload verification failure paths. Uncovered lines for this file drop from 57 to 21 (lcov, line+branch). Remaining lines are either dead code given existing invariants (the per-chunk maxPayloadBytes check, verifyGridFs's non-byte-chunk guard, the "stream writer missing" guard) or depend on timing-sensitive races with no exposed test seam (recoverStaleUpload's current-state guard, the exact instant a GridFS write signals backpressorem). A test also surfaced a real defect, reported separately: finishGarbage's FileNotFound-swallow regex never matches the mongodb driver's actual "File not found for id X" message, so a benign already-deleted-file race during cleanup always rethrows instead of being swallowed. Co-Authored-By: Claude Fable 5.1 --- .../__tests/mongo/MongoPayloadStore.test.ts | 593 +++++++++++++++++- 1 file changed, 592 insertions(+), 1 deletion(-) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts index 9b2a906a7..ab2dfda1f 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts @@ -1,7 +1,13 @@ import { createHash } from 'node:crypto' import { setTimeout as delay } from 'node:timers/promises' +import type { Binary } from 'mongodb' import { getAdmissionStorage } from '../../storage/AdmissionStorage.js' -import { bootstrapMongoOverlay } from '../../storage/mongo/MongoSchema.js' +import { + bootstrapMongoOverlay, + encodeMongoUint64, + mongoChainKey, + mongoRecordKey +} from '../../storage/mongo/MongoSchema.js' import { MongoPayloadStore } from '../../storage/mongo/MongoPayloadStore.js' import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' @@ -15,6 +21,37 @@ const bytes = async function* ( const digest = (value: Uint8Array): string => createHash('sha256').update(value).digest('hex') +/** + * A minimal AbortSignal-shaped test double whose `aborted` flag can be + * flipped after construction (a real AbortSignal cannot). It implements the + * only members MongoPayloadStore reads from a signal: the synchronous + * `aborted`/`reason` properties, plus the two EventTarget methods raceAbort + * registers, so it is safe to pass anywhere `AbortSignal` is accepted. + */ +function fakeAbortSignal(): AbortSignal & { + setAborted: (aborted: boolean) => void + setReason: (reason: unknown) => void +} { + let abortedFlag = false + let reasonValue: unknown + return { + get aborted() { + return abortedFlag + }, + get reason() { + return reasonValue + }, + setAborted(aborted: boolean) { + abortedFlag = aborted + }, + setReason(reason: unknown) { + reasonValue = reason + }, + addEventListener: () => undefined, + removeEventListener: () => undefined + } as unknown as AbortSignal & { setAborted: (aborted: boolean) => void; setReason: (reason: unknown) => void } +} + describe('MongoPayloadStore', () => { let fixture: MongoReplicaFixture let store: MongoPayloadStore @@ -566,6 +603,28 @@ describe('MongoPayloadStore', () => { expect(await fixture.db.collection('overlay_payloads').countDocuments({ digest: hash })).toBe(2) }) + test('many concurrent first-time publishers of the same digest still converge on one ready payload', async () => { + const weak = await fixture.connect({ retryWrites: false }) + const racer = new MongoPayloadStore(weak.db(fixture.db.databaseName), fixture.scope) + const content = Buffer.from('many-way-overlap-same-digest') + const hash = digest(content) + const publish = async (): Promise => + racer.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + const results = await Promise.all(Array.from({ length: 12 }, publish)) + for (const result of results) expect(result).toEqual(results[0]) + expect( + await fixture.db + .collection('overlay_payloads') + .countDocuments({ kind: 'outbox-data', digest: hash, state: 'ready' }) + ).toBe(1) + await weak.close() + }) + test('recovers an upload that crashed after GridFS publication and before ready', async () => { const content = Buffer.alloc(300 * 1024, 0x71) const hash = digest(content) @@ -904,4 +963,536 @@ describe('MongoPayloadStore', () => { }) ).rejects.toThrow('inline BSON document exceeds safety ceiling') }, 30000) + + test('re-adding a pin slot with the identical unexpired expiry performs no write', async () => { + const fixedNow = new Date('2030-01-01T00:00:00.000Z') + const stable = new MongoPayloadStore(fixture.db, fixture.scope, { now: () => fixedNow }) + const content = Buffer.from('pin-identical-expiry') + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await stable.publish({ ...payload, byteLength: String(content.byteLength), bytes: bytes(content) }) + const expiresAt = new Date(fixedNow.getTime() + 60_000) + const pin = { + scope: fixture.scope, + payload, + ownerKind: 'pin' as const, + ownerId: 'pin-identical-expiry', + slot: '0', + expiresAt + } + const session = fixture.client.startSession() + await session.withTransaction(async () => { + await stable.addReference(session, pin) + }) + const first = await fixture.db + .collection('overlay_payload_references') + .findOne({ ownerId: 'pin-identical-expiry' }) + await session.withTransaction(async () => { + await stable.addReference(session, pin) + }) + const second = await fixture.db + .collection('overlay_payload_references') + .findOne({ ownerId: 'pin-identical-expiry' }) + expect(second?.updatedAt).toEqual(first?.updatedAt) + expect(second?.expiresAt).toEqual(expiresAt) + await session.endSession() + }) + + test('claimGarbage on a payload that was never published returns false without side effects', async () => { + const missing = { kind: 'outbox-data' as const, digest: digest(Buffer.from('never-published')) } + const session = fixture.client.startSession() + await session.withTransaction(async () => { + expect(await store.claimGarbage(session, missing)).toBe(false) + }) + await session.endSession() + }) + + test('invokes the afterDeleteClaim hook exactly when a GC claim succeeds', async () => { + let calls = 0 + const hooked = new MongoPayloadStore(fixture.db, fixture.scope, { + hooks: { + afterDeleteClaim: () => { + calls += 1 + } + } + }) + const content = Buffer.from('hook-delete-claim') + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await hooked.publish({ ...payload, byteLength: String(content.byteLength), bytes: bytes(content) }) + const session = fixture.client.startSession() + await session.withTransaction(async () => { + expect(await hooked.claimGarbage(session, payload)).toBe(true) + }) + await session.endSession() + expect(calls).toBe(1) + }) + + test('finishGarbage preserves a GridFS file whose ownership metadata has since changed', async () => { + const content = Buffer.alloc(300 * 1024, 0x11) + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024) + }) + const payloads = fixture.db.collection('overlay_payloads') + const row = await payloads.findOne({ digest: hash, kind: 'outbox-data' }) + expect(row?.fileId).toBeDefined() + const session = fixture.client.startSession() + await session.withTransaction(async () => { + expect(await store.claimGarbage(session, payload)).toBe(true) + }) + await session.endSession() + await fixture.db + .collection('overlayPayloads.files') + .updateOne({ _id: row?.fileId }, { $set: { 'metadata.ownerId': 'someone-else' } }) + expect(await store.finishGarbage(payload)).toBe(true) + expect((await payloads.findOne({ _id: row?._id }))?.state).toBe('deleted') + expect( + await fixture.db.collection('overlayPayloads.files').countDocuments({ _id: row?.fileId }) + ).toBe(1) + }) + + test('rejects publication when the upload lease is stolen just before the ready CAS', async () => { + const content = Buffer.from('stolen-before-ready') + const hash = digest(content) + const fencing = new MongoPayloadStore(fixture.db, fixture.scope, { + hooks: { + beforeReadyCas: async () => { + await fixture.db + .collection('overlay_payloads') + .updateOne( + { kind: 'outbox-data', digest: hash }, + { $set: { guard: 'stolen-guard', ownerId: 'stolen-owner' } } + ) + } + } + }) + await expect( + fencing.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + ).rejects.toThrow('fenced before publication') + }) + + test('rejects a wrong explicit txid on an otherwise-correct raw-transaction upload', async () => { + const content = Buffer.from('02000000000000000001', 'hex') + const hash = digest(content) + await expect( + store.publish({ + kind: 'raw-transaction', + digest: hash, + byteLength: String(content.byteLength), + txid: 'ab'.repeat(32), + bytes: bytes(content) + }) + ).rejects.toThrow('does not match bytes') + }) + + test('rejects a wrong explicit txid when the referenced content is already ready', async () => { + const content = Buffer.from('03000000000000000002', 'hex') + const hash = digest(content) + await store.publish({ + kind: 'raw-transaction', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content) + }) + await expect( + store.publish({ + kind: 'raw-transaction', + digest: hash, + byteLength: String(content.byteLength), + txid: 'cd'.repeat(32), + bytes: bytes(content) + }) + ).rejects.toThrow('does not match bytes') + }) + + test('rejects with a generic message when an aborted signal carries no explicit reason', async () => { + const signal = fakeAbortSignal() + const content = Buffer.from('fake-signal-no-reason') + const chunked: AsyncIterable = { + [Symbol.asyncIterator]() { + let sent = false + return { + next: async () => { + if (sent) return { done: true as const, value: undefined } + sent = true + signal.setAborted(true) + return { done: false as const, value: content } + } + } + } + } + await expect( + store.publish({ + kind: 'outbox-data', + digest: digest(content), + byteLength: String(content.byteLength), + bytes: chunked, + signal + }) + ).rejects.toThrow('Mongo payload upload aborted') + }) + + test('rejects with the signal reason when a chunk observes an already-aborted signal that carries one', async () => { + const signal = fakeAbortSignal() + const reason = new Error('fake-signal-explicit-reason') + const content = Buffer.from('fake-signal-with-reason') + const chunked: AsyncIterable = { + [Symbol.asyncIterator]() { + let sent = false + return { + next: async () => { + if (sent) return { done: true as const, value: undefined } + sent = true + signal.setReason(reason) + signal.setAborted(true) + return { done: false as const, value: content } + } + } + } + } + await expect( + store.publish({ + kind: 'outbox-data', + digest: digest(content), + byteLength: String(content.byteLength), + bytes: chunked, + signal + }) + ).rejects.toBe(reason) + }) + + test('claimGarbage falls back to a generic message when the operation signal reason is unset', async () => { + const signal = fakeAbortSignal() + signal.setAborted(true) + const session = fixture.client.startSession() + await expect( + store.claimGarbage( + session, + { kind: 'outbox-data', digest: digest(Buffer.from('fake-signal-claim')) }, + { signal } + ) + ).rejects.toThrow('Mongo payload operation aborted') + await session.endSession() + }) + + test('rejects staging when the upload lease is stolen mid-stream', async () => { + const content = Buffer.alloc(300 * 1024, 0x22) + const hash = digest(content) + let stolen = false + const chunked = async function* (): AsyncIterable { + const chunkSize = 64 * 1024 + for (let offset = 0; offset < content.byteLength; offset += chunkSize) { + yield content.subarray(offset, offset + chunkSize) + if (!stolen && offset > 0) { + stolen = true + await fixture.db + .collection('overlay_payloads') + .updateOne( + { kind: 'outbox-data', digest: hash }, + { $set: { guard: 'mid-stream-steal', ownerId: 'mid-stream-steal' } } + ) + } + } + } + await expect( + store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: chunked() + }) + ).rejects.toThrow('fenced before staging') + }) + + test('invokes GridFS crash-boundary hooks and rejects when the staged file state changes before publication', async () => { + const content = Buffer.alloc(300 * 1024, 0x33) + const hash = digest(content) + const hookCalls: string[] = [] + const hooked = new MongoPayloadStore(fixture.db, fixture.scope, { + hooks: { + afterGridFsUploaded: async () => { + hookCalls.push('afterGridFsUploaded') + await fixture.db + .collection('overlayPayloads.files') + .updateOne({ 'metadata.digest': hash }, { $set: { 'metadata.state': 'corrupted' } }) + }, + afterGridFsPublished: () => { + hookCalls.push('afterGridFsPublished') + }, + beforeReadyCas: () => { + hookCalls.push('beforeReadyCas') + } + } + }) + await expect( + hooked.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024) + }) + ).rejects.toThrow('staged file was lost') + expect(hookCalls).toEqual(['afterGridFsUploaded']) + }) + + test('invokes every GridFS crash-boundary hook in order on a successful large upload', async () => { + const content = Buffer.alloc(300 * 1024, 0x44) + const hash = digest(content) + const hookCalls: string[] = [] + const hooked = new MongoPayloadStore(fixture.db, fixture.scope, { + hooks: { + afterGridFsUploaded: () => { + hookCalls.push('afterGridFsUploaded') + }, + afterGridFsPublished: () => { + hookCalls.push('afterGridFsPublished') + }, + beforeReadyCas: () => { + hookCalls.push('beforeReadyCas') + } + } + }) + await expect( + hooked.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024) + }) + ).resolves.toMatchObject({ digest: hash }) + expect(hookCalls).toEqual(['afterGridFsUploaded', 'afterGridFsPublished', 'beforeReadyCas']) + }) + + test('abandonUpload does not delete a file a fenced winner already published', async () => { + const content = Buffer.alloc(300 * 1024, 0x55) + const hash = digest(content) + const crashing = new MongoPayloadStore(fixture.db, fixture.scope, { + hooks: { + beforeReadyCas: async () => { + const row = await fixture.db + .collection('overlay_payloads') + .findOne({ kind: 'outbox-data', digest: hash }) + await fixture.db + .collection('overlayPayloads.files') + .updateOne({ _id: row?.fileId }, { $set: { 'metadata.state': 'published' } }) + throw new Error('simulated crash after independent publish') + } + } + }) + await expect( + crashing.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024) + }) + ).rejects.toThrow('simulated crash after independent publish') + const payloadRow = await fixture.db + .collection('overlay_payloads') + .findOne({ kind: 'outbox-data', digest: hash }) + expect(payloadRow?.state).toBe('uploading') + expect(payloadRow?.fileId).toBeDefined() + expect( + await fixture.db.collection('overlayPayloads.files').countDocuments({ _id: payloadRow?.fileId }) + ).toBe(1) + }) + + test('a digest mismatch detected after spilling to GridFS retires the orphaned file', async () => { + const content = Buffer.alloc(300 * 1024, 0x66) + const wrongDigest = digest(Buffer.from('wrong-declared-digest')) + await expect( + store.publish({ + kind: 'outbox-data', + digest: wrongDigest, + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024) + }) + ).rejects.toThrow('digest or declared length mismatch') + const row = await fixture.db + .collection('overlay_payloads') + .findOne({ kind: 'outbox-data', digest: wrongDigest }) + expect(row?.state).toBe('deleted') + expect(row?.retiredFileId).toBeDefined() + expect( + await fixture.db.collection('overlayPayloads.files').countDocuments({ _id: row?.retiredFileId }) + ).toBe(0) + }) + + test('recovers a stale inline upload that has no GridFS file by marking it deleted', async () => { + const now = new Date() + const hash = digest(Buffer.from('stale-inline-upload')) + const payloadId = `stale-inline-${hash}` + await fixture.db.collection('overlay_payloads').insertOne({ + _id: payloadId, + schemaVersion: 1, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash, + kind: 'outbox-data', + digest: hash, + byteLength: encodeMongoUint64('5'), + state: 'uploading', + guard: 'stale-owner', + ownerNodeId: fixture.scope.nodeId, + ownerId: 'stale-owner', + fencingToken: encodeMongoUint64('0'), + leaseUntil: new Date(Date.now() - 1000), + createdAt: now, + updatedAt: now + }) + await store.recoverUploads() + const row = await fixture.db.collection('overlay_payloads').findOne({ _id: payloadId }) + expect(row?.state).toBe('deleted') + expect(row?.fileId).toBeUndefined() + expect(row?.retiredFileId).toBeUndefined() + }) + + test('retires an orphaned GridFS file when a fresh publish reclaims an expired crashed reservation', async () => { + const donorContent = Buffer.alloc(300 * 1024, 0xdd) + const donorHash = digest(donorContent) + await store.publish({ + kind: 'outbox-data', + digest: donorHash, + byteLength: String(donorContent.byteLength), + bytes: bytes(donorContent, 64 * 1024) + }) + const donorRow = await fixture.db + .collection('overlay_payloads') + .findOne({ kind: 'outbox-data', digest: donorHash }) + const donorFile = await fixture.db + .collection('overlayPayloads.files') + .findOne({ _id: donorRow?.fileId }) + expect(donorFile).not.toBeNull() + + const crashedContent = Buffer.from('crashed-reservation') + const crashedHash = digest(crashedContent) + const crashedOwner = (donorFile as { metadata: { ownerId: string } }).metadata.ownerId + const crashedFence = (donorFile as { metadata: { fencingToken: string } }).metadata.fencingToken + await fixture.db.collection('overlay_payloads').insertOne({ + _id: mongoRecordKey(mongoChainKey(fixture.scope), 'outbox-data', crashedHash), + schemaVersion: 1, + network: fixture.scope.network, + genesisHash: fixture.scope.genesisHash, + kind: 'outbox-data', + digest: crashedHash, + byteLength: encodeMongoUint64(String(crashedContent.byteLength)), + state: 'uploading', + guard: crashedOwner, + ownerNodeId: fixture.scope.nodeId, + ownerId: crashedOwner, + fencingToken: crashedFence, + fileId: donorFile?._id, + leaseUntil: new Date(Date.now() - 1000), + createdAt: new Date(), + updatedAt: new Date() + }) + await expect( + store.publish({ + kind: 'outbox-data', + digest: crashedHash, + byteLength: String(crashedContent.byteLength), + bytes: bytes(crashedContent) + }) + ).resolves.toMatchObject({ digest: crashedHash }) + expect( + await fixture.db.collection('overlayPayloads.files').countDocuments({ _id: donorFile?._id }) + ).toBe(0) + }) + + test('claimGarbage rejects immediately with the operation signal reason when already aborted', async () => { + const controller = new AbortController() + const reason = new Error('operation cancelled up front') + controller.abort(reason) + const session = fixture.client.startSession() + await expect( + store.claimGarbage( + session, + { kind: 'outbox-data', digest: digest(Buffer.from('pre-aborted')) }, + { signal: controller.signal } + ) + ).rejects.toBe(reason) + await session.endSession() + }) + + test('publishes normally with a live but never-aborted signal attached to a GridFS upload', async () => { + const content = Buffer.alloc(300 * 1024, 0x77) + const hash = digest(content) + const controller = new AbortController() + await expect( + store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024), + signal: controller.signal + }) + ).resolves.toMatchObject({ digest: hash }) + }) + + test('propagates a genuine iterator failure through raceAbort when a live signal is attached', async () => { + const controller = new AbortController() + const failing: AsyncIterable = { + [Symbol.asyncIterator]() { + return { + next: async () => { + throw new Error('genuine-iterator-failure') + } + } + } + } + await expect( + store.publish({ + kind: 'outbox-data', + digest: digest(Buffer.from('genuine-iterator-failure-payload')), + byteLength: '1', + bytes: failing, + signal: controller.signal + }) + ).rejects.toThrow('genuine-iterator-failure') + expect(controller.signal.aborted).toBe(false) + }) + + test('rejects a staged GridFS upload whose content was corrupted without changing its declared length', async () => { + const content = Buffer.alloc(300 * 1024, 0xaa) + const hash = digest(content) + await store.publish({ + kind: 'outbox-data', + digest: hash, + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024) + }) + const payloads = fixture.db.collection('overlay_payloads') + const row = await payloads.findOne({ kind: 'outbox-data', digest: hash }) + expect(row?.fileId).toBeDefined() + const chunk = await fixture.db + .collection('overlayPayloads.chunks') + .findOne({ files_id: row?.fileId, n: 0 }) + const originalData = chunk?.data as Buffer | Binary + const originalLength = Buffer.isBuffer(originalData) ? originalData.length : originalData.length() + await fixture.db + .collection('overlayPayloads.chunks') + .updateOne({ _id: chunk?._id }, { $set: { data: Buffer.alloc(originalLength, 0xcc) } }) + await payloads.updateOne({ _id: row?._id }, [ + { + $set: { + state: 'uploading', + leaseUntil: { $dateSubtract: { startDate: '$$NOW', unit: 'second', amount: 1 } } + } + } + ]) + await store.recoverUploads() + const recovered = await payloads.findOne({ _id: row?._id }) + expect(recovered?.state).toBe('deleted') + expect( + await fixture.db.collection('overlayPayloads.files').countDocuments({ _id: row?.fileId }) + ).toBe(0) + }) }) From c4b4106c60be30addc78494e7672be665e3c0383 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 20:20:20 -0500 Subject: [PATCH 164/168] test(overlay): pin MongoTransactionRunner claim races and reconcile edges Adds real jest tests for previously-uncovered MongoTransactionRunner branches: many concurrent fresh claims on the same operation id (forces the duplicate-key insert race and the "lost the claim" observer path), a claim-row insert failure whose error code is not a duplicate key (must propagate, not be swallowed), reconcile() with no attemptId against a never-created operation (the "unlocated" fallback), reconcile() with a mismatched attemptId against a genuinely pending claim, and commit failures whose deadline elapses mid-retry-loop (the budget-aborted break, verified via a real blocked commitTransaction). Uncovered lines for this file drop from 18 to 9 (lcov, line+branch). Remaining lines depend on races or invariants not reachable from the public API: Budget's manual deadline check is shadowed by the earlier throwIfAborted() once the matching timer fires; abort()'s "already out of transaction" branch has no path that reaches it before the catch that calls it; the post-insert re-read uses a majority/primary read right after an acknowledged write, so it can't observe a miss without breaking the replica set's own consistency guarantee; and the ownership-fencing mismatches and the "operation row disappeared" branches require deleting a submission-operation row, which nothing in this class ever does. Co-Authored-By: Claude Fable 5.1 --- .../mongo/MongoTransactionRunner.test.ts | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts index 03cc1828d..a545db134 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoTransactionRunner.test.ts @@ -337,4 +337,74 @@ describe('Mongo transaction boundary on three data-bearing WiredTiger members', expect(await restarted.reconcile(input.key, undefined, { timeoutMS: 15000 })).toEqual({ state: 'committed', receipt: input.receipt }) expect(await fixture.db.collection('test_effects').countDocuments({ operationId: input.key.operationId })).toBe(1) }, 60000) + + test('many concurrent fresh claims on the same key converge without corrupting the operation row', async () => { + const racer = new MongoTransactionRunner(fixture.db, fixture.scope, { maxBodyAttempts: 3, maxCommitAttempts: 2 }) + runners.push(racer) + const input = request() + const outcomes = await Promise.allSettled( + Array.from({ length: 10 }, async () => await racer.run(input, async () => { throw new Error('always fails') })) + ) + expect(outcomes).toHaveLength(10) + // Every racer either loses the claim outright (observed as still pending, + // an ordinary fulfilled result) or wins a claim and then rejects, since + // the body always throws; a body never legitimately reaches 'committed'. + let rejected = 0 + for (const outcome of outcomes) { + if (outcome.status === 'rejected') { + rejected += 1 + expect((outcome.reason as Error).message).toMatch(/always fails|attempt limit reached/) + } else { + expect(outcome.value.state).toBe('pending') + } + } + expect(rejected).toBeGreaterThan(0) + expect(await racer.reconcile(input.key)).toEqual({ state: 'aborted' }) + }, 30000) + + test('propagates a claim-row insert failure whose error code is not a duplicate key', async () => { + const input = request() + await fixture.failCommands({ failCommands: ['insert'], errorCode: 8 }, 1) + try { + await expect(runner.run(input, async () => {})).rejects.toMatchObject({ code: 8 }) + } finally { + await fixture.disableFailPoint() + } + }) + + test('reconcile with no attemptId falls back to "unlocated" for a never-created operation', async () => { + const input = request() + expect(await runner.reconcile(input.key)).toEqual({ state: 'pending', attemptId: 'unlocated' }) + }) + + test('reconcile with a mismatched attemptId returns pending without touching the retained attempt', async () => { + const input = request() + let resume!: () => void + const barrier = new Promise(resolve => { resume = resolve }) + let entered!: () => void + const started = new Promise(resolve => { entered = resolve }) + const pending = runner.run(input, async () => { entered(); await barrier }) + await started + await expect(runner.reconcile(input.key, 'not-the-real-attempt-id')).resolves.toEqual({ + state: 'pending', + attemptId: 'not-the-real-attempt-id' + }) + resume() + expect((await pending).state).toBe('committed') + }) + + test('commit failures across an elapsing deadline break out of the retry loop and report pending', async () => { + const shortDeadline = new MongoTransactionRunner(fixture.db, fixture.scope, { maxCommitAttempts: 5 }) + runners.push(shortDeadline) + const input = request() + await fixture.failCommands( + { failCommands: ['commitTransaction'], errorCode: 91, blockConnection: true, blockTimeMS: 80 }, + 5 + ) + const result = await shortDeadline.run(input, async () => {}, { timeoutMS: 120 }) + expect(result.state).toBe('pending') + await fixture.disableFailPoint() + const attemptId = result.state === 'pending' ? result.attemptId : undefined + expect((await shortDeadline.reconcile(input.key, attemptId)).state).toBe('committed') + }, 15000) }) From 2f22e39547e62316250c4feb63cc57d6b7b5a996 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 20:20:31 -0500 Subject: [PATCH 165/168] test(overlay): pin MongoSchema bootstrap idempotency and failure paths Adds real jest tests for previously-uncovered MongoSchema branches: concurrent bootstraps of a fresh database (races the NamespaceExists and duplicate-ledger swallow paths in createCollectionIfMissing, ensureGridFs, and ensureLedger), a concurrent re-bootstrap after dropping just the GridFS bucket collections, an existing collection whose validator matches but omits an explicit collation (the collation-compatible-by-omission branch), an index-creation failure whose error code is not one of the benign conflict codes, and a ledger insert failure whose error code is not a duplicate key (both must propagate rather than being swallowed). Uncovered lines for this file drop from 11 to 10 deterministically; the concurrent-bootstrap races additionally land on the NamespaceExists swallow branches in some runs but not every run, since they depend on genuine scheduling of concurrent createCollection calls. The remaining lines are dead code given the surrounding invariants (decodeMongoUint64's re-encode check, the unused `string()` default length and `allowAdditional` option on module-private helpers no call site ever exercises) or require racing a second writer against transactionalProbe's own read-then-CAS window, which was not attempted. Co-Authored-By: Claude Fable 5.1 --- .../src/__tests/mongo/MongoSchema.test.ts | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts index 89fb0611d..cc4a76757 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoSchema.test.ts @@ -371,4 +371,74 @@ describe('Mongo schema bootstrap', () => { `Incompatible Mongo Overlay index for ${MongoCollectionNames.payloadReferences}:pin_expiry` ) }) + + test('concurrent bootstraps of a fresh database converge without duplicate-namespace or duplicate-ledger errors', async () => { + const database = fixture.client.db(`overlay_s02_concurrent_${Date.now()}`) + const results = await Promise.all( + Array.from({ length: 6 }, async () => await bootstrapMongoOverlay(database, fixture.scope)) + ) + for (const result of results) { + expect(result.topology).toBe('replica-set') + expect(result.schemaVersion).toBe(1) + } + expect( + await database.listCollections({ name: `${MongoGridFsBucketName}.files` }).hasNext() + ).toBe(true) + expect( + await database.collection(MongoCollectionNames.schema).countDocuments({ _id: mongoNodeKey(fixture.scope) }) + ).toBe(1) + }, 30000) + + test('concurrent re-bootstraps race the GridFS bucket namespace once its collections are dropped', async () => { + const database = fixture.client.db(`overlay_s02_gridfs_race_${Date.now()}`) + await bootstrapMongoOverlay(database, fixture.scope) + await database.collection(`${MongoGridFsBucketName}.files`).drop() + await database.collection(`${MongoGridFsBucketName}.chunks`).drop() + const results = await Promise.all( + Array.from({ length: 8 }, async () => await bootstrapMongoOverlay(database, fixture.scope)) + ) + for (const result of results) expect(result.topology).toBe('replica-set') + expect( + await database.listCollections({ name: `${MongoGridFsBucketName}.files` }).hasNext() + ).toBe(true) + expect( + await database.listCollections({ name: `${MongoGridFsBucketName}.chunks` }).hasNext() + ).toBe(true) + }, 30000) + + test('accepts an existing collection whose validator matches but omits an explicit collation', async () => { + const definition = MongoCollectionDefinitions.find( + item => item.name === MongoCollectionNames.outputs + ) + if (definition === undefined) throw new Error('Missing outputs schema definition') + const database = fixture.client.db(`overlay_s02_no_collation_${Date.now()}`) + await database.createCollection(MongoCollectionNames.outputs, { + validator: definition.validator, + validationLevel: 'strict', + validationAction: 'error' + }) + await expect(bootstrapMongoOverlay(database, fixture.scope)).resolves.toMatchObject({ + topology: 'replica-set' + }) + }, 30000) + + test('propagates an index-creation failure whose error code is not a benign conflict', async () => { + const database = fixture.client.db(`overlay_s02_index_failure_${Date.now()}`) + await fixture.failCommands({ failCommands: ['createIndexes'], errorCode: 50 }) + try { + await expect(bootstrapMongoOverlay(database, fixture.scope)).rejects.toMatchObject({ code: 50 }) + } finally { + await fixture.disableFailPoint() + } + }, 30000) + + test('propagates a ledger insert failure whose error code is not a duplicate key', async () => { + const database = fixture.client.db(`overlay_s02_ledger_insert_failure_${Date.now()}`) + await fixture.failCommands({ failCommands: ['insert'], errorCode: 50 }) + try { + await expect(bootstrapMongoOverlay(database, fixture.scope)).rejects.toMatchObject({ code: 50 }) + } finally { + await fixture.disableFailPoint() + } + }, 30000) }) From 24cc89b90e5b0a980a1a765aaaa9ff5e2d12de44 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 20:48:17 -0500 Subject: [PATCH 166/168] fix(overlay-mongo): tolerate an already-deleted GridFS file in finishGarbage finishGarbage is documented as safe to retry, but it only swallowed errors matching /FileNotFound/. The MongoDB driver reports a missing file as MongoRuntimeError "File not found for id ", which never matched, so a finisher that lost the race to a concurrent retry rethrew and left the payload row in 'deleting'. Match the driver's actual message. Co-Authored-By: Claude Fable 5.1 --- .../__tests/mongo/MongoPayloadStore.test.ts | 36 ++++++++++++++++++- .../src/storage/mongo/MongoPayloadStore.ts | 4 ++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts index ab2dfda1f..d9900ed1f 100644 --- a/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts +++ b/packages/overlays/overlay/src/__tests/mongo/MongoPayloadStore.test.ts @@ -1,6 +1,6 @@ import { createHash } from 'node:crypto' import { setTimeout as delay } from 'node:timers/promises' -import type { Binary } from 'mongodb' +import type { Binary, GridFSBucket } from 'mongodb' import { getAdmissionStorage } from '../../storage/AdmissionStorage.js' import { bootstrapMongoOverlay, @@ -1055,6 +1055,40 @@ describe('MongoPayloadStore', () => { ).toBe(1) }) + test('finishGarbage tolerates a concurrent retry that already deleted the GridFS file', async () => { + const content = Buffer.alloc(300 * 1024, 0x12) + const hash = digest(content) + const payload = { kind: 'outbox-data' as const, digest: hash } + await store.publish({ + ...payload, + byteLength: String(content.byteLength), + bytes: bytes(content, 64 * 1024) + }) + const payloads = fixture.db.collection('overlay_payloads') + const row = await payloads.findOne({ digest: hash, kind: 'outbox-data' }) + expect(row?.fileId).toBeDefined() + const session = fixture.client.startSession() + await session.withTransaction(async () => { + expect(await store.claimGarbage(session, payload)).toBe(true) + }) + await session.endSession() + // A competing finisher removes the file after this one's ownership read, so + // the real driver raises its own "File not found for id" error here. + const bucket = (store as unknown as { bucket: GridFSBucket }).bucket + const realDelete = bucket.delete.bind(bucket) + const deleteSpy = jest.spyOn(bucket, 'delete').mockImplementationOnce(async id => { + await realDelete(id) + await realDelete(id) + }) + expect(await store.finishGarbage(payload)).toBe(true) + expect(deleteSpy).toHaveBeenCalledTimes(1) + expect((await payloads.findOne({ _id: row?._id }))?.state).toBe('deleted') + expect( + await fixture.db.collection('overlayPayloads.files').countDocuments({ _id: row?.fileId }) + ).toBe(0) + deleteSpy.mockRestore() + }) + test('rejects publication when the upload lease is stolen just before the ready CAS', async () => { const content = Buffer.from('stolen-before-ready') const hash = digest(content) diff --git a/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts b/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts index 1ee0d309f..ae1d8e0fe 100644 --- a/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts +++ b/packages/overlays/overlay/src/storage/mongo/MongoPayloadStore.ts @@ -438,7 +438,9 @@ export class MongoPayloadStore { }) if (owned !== null) { await this.bucket.delete(record.fileId).catch(error => { - if (!(error instanceof Error) || !/FileNotFound/.test(error.message)) throw error + // The driver reports an already-removed file as MongoRuntimeError + // "File not found for id "; a concurrent finisher got there first. + if (!(error instanceof Error) || !/File not found/i.test(error.message)) throw error }) } } From 0f5cb960eccc616d8f0618c047432be4e51196d1 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 21:52:25 -0500 Subject: [PATCH 167/168] fix(overlay): reconcile BASM heights where the topic admitted nothing BRC-136 defines an empty topic at a height as k = 0 with R = 32 zero bytes, and rebuildTopicAnchorChain anchors every height that way. The reconciler accepted such an anchor and then requested a compound Merkle path for an empty txid list, which BASMRemote rejects ("At least one BASM proof txid is required"). Any node behind a peer aborted at the first height the topic admitted nothing, which is most heights. After the root and count check, an empty admitted list has nothing to bind or fetch: count the height as checked and skip the proof and raw transaction round trip. It diverges only when this node admitted transactions at that height. A zero-count anchor with a non-zero root is still rejected. Co-Authored-By: Claude Fable 5.1 --- packages/overlays/overlay/CHANGELOG.md | 4 +- packages/overlays/overlay/src/Engine.ts | 8 ++ .../src/__tests/BASMEmptyBlock.test.ts | 93 +++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 packages/overlays/overlay/src/__tests/BASMEmptyBlock.test.ts diff --git a/packages/overlays/overlay/CHANGELOG.md b/packages/overlays/overlay/CHANGELOG.md index 1a4a22f96..0f74ad78b 100644 --- a/packages/overlays/overlay/CHANGELOG.md +++ b/packages/overlays/overlay/CHANGELOG.md @@ -34,7 +34,9 @@ All notable changes to this project will be documented in this file. The format chain-tracker root/height rather than coinbase maturity, so BASM admission submits in the `historical-tx-no-spv` mode and topic managers that branch on the submission mode observe it for BASM-admitted transactions; the public - `historical-tx` mode keeps full SPV verification. Historical sync + `historical-tx` mode keeps full SPV verification. Heights where the topic + admitted nothing (BRC-136 `k = 0`, zero BASM root) are checked without a proof + request instead of aborting the sync. Historical sync refuses an untrusted TAC prefix; durable bootstrap, chunked recovery and topic status remain pending. No storage migration or automatic sync activation is included. diff --git a/packages/overlays/overlay/src/Engine.ts b/packages/overlays/overlay/src/Engine.ts index 9f9982ce4..2f0481ee0 100644 --- a/packages/overlays/overlay/src/Engine.ts +++ b/packages/overlays/overlay/src/Engine.ts @@ -1858,6 +1858,14 @@ export class Engine { } const localAdmitted = await this.storage.findAdmittedTransactionsForBlock?.(topic, remoteAnchor.blockHeight, remoteAnchor.blockHash) ?? [] + // BRC-136: an empty topic at a height has k = 0 and R = 32 zero bytes, which + // the root/count check above has already enforced. There is no transaction + // to bind to a proof or to fetch, so the height is checked without a proof + // round trip; it only diverges when this node admitted something there. + if (admittedResponse.admitted.length === 0) { + if (localAdmitted.length > 0) report.status = 'diverged' + return + } const localTxids = new Set(localAdmitted.map(item => item.txid)) const missingTxids = admittedResponse.admitted .map(item => item.txid) diff --git a/packages/overlays/overlay/src/__tests/BASMEmptyBlock.test.ts b/packages/overlays/overlay/src/__tests/BASMEmptyBlock.test.ts new file mode 100644 index 000000000..efa6f4f3a --- /dev/null +++ b/packages/overlays/overlay/src/__tests/BASMEmptyBlock.test.ts @@ -0,0 +1,93 @@ +import { BASM_ZERO_HASH, computeBasmRoot, computeTac } from '../BASM' +import type { TopicBlockAnchor } from '../BASM' +import { blockHash, blockHeight, fixture, topic } from './BASMReconciliationFixture' + +afterEach(() => jest.restoreAllMocks()) + +// BRC-136: "Empty topic at this height. If k = 0, then R = 0x00…00 (32 zero bytes)." +// A peer anchors every height, so most anchors a lagging node sees admit nothing. +describe('BASM reconciliation of heights where the topic admitted nothing', () => { + const localHeight = blockHeight - 1 + const localHash = 'cd'.repeat(32) + + function emptyHeightFixture() { + const f = fixture() + const localAnchor: TopicBlockAnchor = { + topic, + blockHeight: localHeight, + blockHash: localHash, + basmRoot: BASM_ZERO_HASH, + admittedCount: 0, + tac: computeTac(BASM_ZERO_HASH, localHash, BASM_ZERO_HASH) + } + const emptyAnchor: TopicBlockAnchor = { + topic, + blockHeight, + blockHash, + basmRoot: BASM_ZERO_HASH, + admittedCount: 0, + tac: computeTac(localAnchor.tac, blockHash, BASM_ZERO_HASH) + } + f.storage.findTopicAnchorTip.mockResolvedValue({ + topic, + blockHeight: localHeight, + tac: localAnchor.tac + }) + f.storage.findTopicBlockAnchor.mockImplementation(async (...args: unknown[]) => + args[1] === localHeight ? localAnchor : undefined + ) + f.engine.topicAnchorHeaderResolver = jest.fn(async (height: number) => ({ + blockHeight: height, + blockHash: height === localHeight ? localHash : blockHash, + merkleRoot: f.root + })) + f.responses['/requestTopicAnchorTip'] = emptyAnchor + f.responses['/requestTopicAnchorRange'] = { topic, anchors: [emptyAnchor] } + f.responses['/requestAdmittedList'] = { topic, blockHeight, blockHash, admitted: [] } + return { f, emptyAnchor } + } + + it('defines the empty BASM root as 32 zero bytes', () => { + expect(computeBasmRoot([])).toBe(BASM_ZERO_HASH) + }) + + it('checks an empty height without requesting a proof or raw transactions', async () => { + const { f } = emptyHeightFixture() + const [report] = await f.engine.startBASMSync() + expect(report.message).toBeUndefined() + expect(report.status).not.toBe('error') + expect(report.status).not.toBe('diverged') + expect(report.checkedHeights).toEqual([blockHeight]) + expect(report.missingTxids).toEqual([]) + const paths = f.requests.map(request => request.path) + expect(paths).not.toContain('/requestCompoundMerklePath') + expect(paths).not.toContain('/requestRawTransactions') + expect(f.submit).not.toHaveBeenCalled() + }) + + it('reports divergence when the peer claims an empty height that is populated locally', async () => { + const { f } = emptyHeightFixture() + f.storage.findAdmittedTransactionsForBlock.mockResolvedValue([ + { txid: f.ids[1], blockIndex: 1 } + ]) + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('diverged') + expect(f.requests.map(request => request.path)).not.toContain('/requestCompoundMerklePath') + expect(f.submit).not.toHaveBeenCalled() + }) + + it('still rejects a zero-count anchor whose root is not the zero hash', async () => { + const { f, emptyAnchor } = emptyHeightFixture() + const forged = { ...emptyAnchor, basmRoot: 'ee'.repeat(32) } + forged.tac = computeTac( + computeTac(BASM_ZERO_HASH, localHash, BASM_ZERO_HASH), + blockHash, + forged.basmRoot + ) + f.responses['/requestTopicAnchorTip'] = forged + f.responses['/requestTopicAnchorRange'] = { topic, anchors: [forged] } + const [report] = await f.engine.startBASMSync() + expect(report.status).toBe('error') + expect(report.message).toContain('inconsistent with its anchor') + }) +}) From 7392dbe012521be2cbcf9f578b5d93e0230b1136 Mon Sep 17 00:00:00 2001 From: Deggen Date: Fri, 18 Sep 2026 22:30:31 -0500 Subject: [PATCH 168/168] fix(overlay): keep admission host methods bound to their storage getOverlayAdmissionHost copied publishAdmissionPayload, enlistedIndexTargets and getHistoryFence off the storage as detached function references. Storage adapters are class instances, so inside MongoOverlayStorage.publishAdmissionPayload `this` became the plain host object and `this.payloads.publish` threw a TypeError: the Engine admission path crashed on the first payload with the real Mongo adapter. Every test reaching that path had mocked the method with jest.fn(), which ignores its receiver. Bind each optional method to the storage. Cover it with a class-based unit test and a real replica-set test that publishes through the host object the Engine uses; both failed before the change. The existing helper test asserted function identity, which is the defect, and now asserts that calls are forwarded with the storage as receiver. Co-Authored-By: Claude Fable 5.1 --- .../overlays/overlay/src/EngineAdmission.ts | 8 +- .../src/__tests/EngineAdmission.test.ts | 18 ++-- .../EngineAdmissionHostBinding.test.ts | 83 +++++++++++++++++++ .../mongo/MongoOverlayStorageHost.test.ts | 38 +++++++++ 4 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 packages/overlays/overlay/src/__tests/EngineAdmissionHostBinding.test.ts create mode 100644 packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageHost.test.ts diff --git a/packages/overlays/overlay/src/EngineAdmission.ts b/packages/overlays/overlay/src/EngineAdmission.ts index f4ffb15ca..630c64197 100644 --- a/packages/overlays/overlay/src/EngineAdmission.ts +++ b/packages/overlays/overlay/src/EngineAdmission.ts @@ -64,9 +64,11 @@ export function getOverlayAdmissionHost(storage: unknown): OverlayAdmissionHost return { admission, admissionScope: { ...scope }, - publishAdmissionPayload: host.publishAdmissionPayload, - enlistedIndexTargets: host.enlistedIndexTargets, - getHistoryFence: host.getHistoryFence + // Adapters are class instances whose methods read instance state, so each + // optional method stays bound to the storage it came from. + publishAdmissionPayload: host.publishAdmissionPayload?.bind(host), + enlistedIndexTargets: host.enlistedIndexTargets?.bind(host), + getHistoryFence: host.getHistoryFence?.bind(host) } } diff --git a/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts b/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts index e80b3c53c..1b788390a 100644 --- a/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts +++ b/packages/overlays/overlay/src/__tests/EngineAdmission.test.ts @@ -109,19 +109,27 @@ describe('Engine overlay admission helpers', () => { const publishAdmissionPayload = jest.fn() const enlistedIndexTargets = jest.fn(() => ['Hello']) const getHistoryFence = jest.fn() - const found = getOverlayAdmissionHost({ + const storage = { admission, admissionScope, publishAdmissionPayload, enlistedIndexTargets, getHistoryFence - }) + } + const found = getOverlayAdmissionHost(storage) expect(found?.admission).toBe(admission) expect(found?.admissionScope).toEqual(admissionScope) expect(found?.admissionScope).not.toBe(admissionScope) - expect(found?.publishAdmissionPayload).toBe(publishAdmissionPayload) - expect(found?.enlistedIndexTargets).toBe(enlistedIndexTargets) - expect(found?.getHistoryFence).toBe(getHistoryFence) + // Host methods are forwarded to the storage and keep it as their receiver. + const payload = { kind: 'raw-tx' as const, bytes: Uint8Array.of(1) } + void found?.publishAdmissionPayload?.(payload) + expect(publishAdmissionPayload).toHaveBeenCalledWith(payload) + expect(publishAdmissionPayload.mock.contexts[0]).toBe(storage) + expect(found?.enlistedIndexTargets?.()).toEqual(['Hello']) + expect(enlistedIndexTargets.mock.contexts[0]).toBe(storage) + void found?.getHistoryFence?.('Hello') + expect(getHistoryFence).toHaveBeenCalledWith('Hello') + expect(getHistoryFence.mock.contexts[0]).toBe(storage) }) test('operation ids stay raw when short and hash when oversized or ill-formed', () => { diff --git a/packages/overlays/overlay/src/__tests/EngineAdmissionHostBinding.test.ts b/packages/overlays/overlay/src/__tests/EngineAdmissionHostBinding.test.ts new file mode 100644 index 000000000..b7a2f9ddc --- /dev/null +++ b/packages/overlays/overlay/src/__tests/EngineAdmissionHostBinding.test.ts @@ -0,0 +1,83 @@ +import type { + AdmissionPayloadRef, + AdmissionStorage, + HistoryFence, + StorageScope +} from '../storage/AdmissionStorage' +import { getOverlayAdmissionHost } from '../EngineAdmission' + +// Storage adapters are classes whose optional host methods read instance state. +// The host object must call them on the adapter, not as detached functions. +class InstanceStateStorage { + public readonly admissionScope: StorageScope = { + network: 'testnet', + genesisHash: '11'.repeat(32), + nodeId: 'binding-node' + } + + public readonly admission: AdmissionStorage + private readonly published: string[] = [] + private readonly targets = ['ls_example'] + private readonly fence: HistoryFence = { chainEpoch: '7', topicHistoryGeneration: '3' } + + constructor(admission: AdmissionStorage) { + this.admission = admission + } + + async publishAdmissionPayload(input: { + kind: AdmissionPayloadRef['kind'] + bytes: Uint8Array + }): Promise { + this.published.push(input.kind) + return { kind: input.kind, digest: 'ab'.repeat(32), byteLength: String(input.bytes.byteLength) } + } + + enlistedIndexTargets(): readonly string[] { + return this.targets + } + + async getHistoryFence(): Promise { + return this.fence + } + + publishedKinds(): readonly string[] { + return this.published + } +} + +const admission = { + protocol: 'overlay-admission-v1', + commitAdmission: jest.fn(), + reconcileAdmission: jest.fn() +} as unknown as AdmissionStorage + +describe('getOverlayAdmissionHost', () => { + it('invokes optional host methods on the storage instance', async () => { + const storage = new InstanceStateStorage(admission) + const host = getOverlayAdmissionHost(storage) + expect(host).toBeDefined() + + const ref = await host?.publishAdmissionPayload?.({ + kind: 'raw-tx', + bytes: Uint8Array.of(1, 2, 3) + }) + expect(ref).toEqual({ kind: 'raw-tx', digest: 'ab'.repeat(32), byteLength: '3' }) + expect(storage.publishedKinds()).toEqual(['raw-tx']) + expect(host?.enlistedIndexTargets?.()).toEqual(['ls_example']) + await expect(host?.getHistoryFence?.('tm_example')).resolves.toEqual({ + chainEpoch: '7', + topicHistoryGeneration: '3' + }) + }) + + it('leaves host methods undefined when the storage does not provide them', () => { + const host = getOverlayAdmissionHost({ + admission, + admissionScope: new InstanceStateStorage(admission).admissionScope + }) + expect(host).toBeDefined() + expect(host?.publishAdmissionPayload).toBeUndefined() + expect(host?.enlistedIndexTargets).toBeUndefined() + expect(host?.getHistoryFence).toBeUndefined() + }) +}) diff --git a/packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageHost.test.ts b/packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageHost.test.ts new file mode 100644 index 000000000..a5dff8c31 --- /dev/null +++ b/packages/overlays/overlay/src/__tests/mongo/MongoOverlayStorageHost.test.ts @@ -0,0 +1,38 @@ +import { createHash } from 'node:crypto' +import { getOverlayAdmissionHost } from '../../EngineAdmission.js' +import { MongoOverlayStorage } from '../../storage/mongo/MongoOverlayStorage.js' +import { bootstrapMongoOverlay } from '../../storage/mongo/MongoSchema.js' +import { createMongoReplicaFixture, type MongoReplicaFixture } from './MongoReplicaFixture.js' + +describe('MongoOverlayStorage as the Engine admission host', () => { + let fixture: MongoReplicaFixture + let storage: MongoOverlayStorage + + beforeAll(async () => { + fixture = await createMongoReplicaFixture() + await bootstrapMongoOverlay(fixture.db, fixture.scope) + storage = new MongoOverlayStorage(fixture.db, fixture.scope) + }, 120000) + + afterAll(async () => { + await fixture.close() + }, 60000) + + test('publishes a payload through the host object the Engine uses', async () => { + const host = getOverlayAdmissionHost(storage) + expect(host?.publishAdmissionPayload).toBeDefined() + const bytes = Uint8Array.of(9, 8, 7, 6) + const ref = await host?.publishAdmissionPayload?.({ kind: 'locking-script', bytes }) + expect(ref).toMatchObject({ + kind: 'locking-script', + digest: createHash('sha256').update(bytes).digest('hex'), + byteLength: '4' + }) + }, 30000) + + test('reads the history fence through the host object', async () => { + const host = getOverlayAdmissionHost(storage) + const fence = await host?.getHistoryFence?.('tm_host_binding') + expect(fence).toEqual(await storage.getHistoryFence('tm_host_binding')) + }, 30000) +})