diff --git a/Cargo.lock b/Cargo.lock index edac76b..ab6631d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,6 +19,18 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -76,11 +88,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb3028782f6bf14a6df987244333d34e6b272b5a40a53e4879ec2dfd82275a3a" dependencies = [ "bitcoin", + "hashbrown", + "serde", ] [[package]] name = "bdk_electrum_streaming" -version = "0.5.5" +version = "0.6.0" dependencies = [ "anyhow", "bdk_chain", @@ -90,6 +104,7 @@ dependencies = [ "futures", "futures-timer", "miniscript", + "serde", "serde_json", "tokio", "tokio-util", @@ -573,6 +588,16 @@ dependencies = [ "r-efi", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "serde", +] + [[package]] name = "hex-conservative" version = "0.2.2" diff --git a/bdk_electrum_streaming/Cargo.toml b/bdk_electrum_streaming/Cargo.toml index 646d1ab..d353c61 100644 --- a/bdk_electrum_streaming/Cargo.toml +++ b/bdk_electrum_streaming/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bdk_electrum_streaming" -version = "0.5.5" +version = "0.6.0" description = "Experimental but sane BDK electrum client by @evanlinjin." license = "MIT OR Apache-2.0" edition = "2021" @@ -13,9 +13,10 @@ readme = "README.md" futures = "0.3" futures-timer = "3" anyhow = "1" -bdk_core = "0.6" +bdk_core = { version = "0.6", features = ["serde"] } miniscript = { version = "12.0.0" } electrum_streaming_client = { version = "0.4" } +serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" tracing = "0.1" diff --git a/bdk_electrum_streaming/src/async_client.rs b/bdk_electrum_streaming/src/async_client.rs index eddfa89..37a6718 100644 --- a/bdk_electrum_streaming/src/async_client.rs +++ b/bdk_electrum_streaming/src/async_client.rs @@ -1,4 +1,4 @@ -//! Yoo +//! Driving [`State`](crate::State) over an async transport. use anyhow::Context; use electrum_streaming_client::{ @@ -146,7 +146,7 @@ where ); }, }; - if let Some(update) = state.advance(&mut req_queue, raw)? { + if let Some(update) = state.poll(&mut req_queue, raw)? { update_tx.unbounded_send(update).map_err(|err| anyhow::anyhow!(err.to_string()))?; } } diff --git a/bdk_electrum_streaming/src/blocking_client.rs b/bdk_electrum_streaming/src/blocking_client.rs index 33908e7..35e7d0d 100644 --- a/bdk_electrum_streaming/src/blocking_client.rs +++ b/bdk_electrum_streaming/src/blocking_client.rs @@ -200,7 +200,7 @@ where }; match action { StateAction::FromServer(raw) => { - if let Some(update) = state.advance(&mut req_queue, raw)? { + if let Some(update) = state.poll(&mut req_queue, raw)? { update_tx .send(update) .map_err(|err| anyhow::anyhow!(err.to_string())) diff --git a/bdk_electrum_streaming/src/cache.rs b/bdk_electrum_streaming/src/cache.rs new file mode 100644 index 0000000..a8844fb --- /dev/null +++ b/bdk_electrum_streaming/src/cache.rs @@ -0,0 +1,427 @@ +use std::{ + collections::{BTreeSet, HashMap}, + sync::Arc, +}; + +use bdk_core::{ + bitcoin::{self, block::Header, BlockHash, Transaction, Txid}, + ConfirmationBlockTime, +}; +use electrum_streaming_client::{request, response, ElectrumScriptHash, ElectrumScriptStatus}; + +/// Everything learned from the server, kept so a reconnect need not ask again. +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub struct Cache { + /// The server's per-script histories. + pub subscriptions: Subscriptions, + + /// What we already hold, so a job knows what it need not ask for. + /// + /// Not persisted: every part of it is in the caller's wallet already, and a second copy + /// would only give the two something to disagree about. Seed it from wallet data instead. + #[serde(skip)] + pub tx_cache: TxCache, + + /// This can be removed once we can place `Header`s in `CheckPoint`s. + pub headers: HashMap, +} + +/// The transaction data a job consults before asking the server for anything. +/// +/// Separate from the rest of [`Cache`] because a caller can rebuild all of it from their own +/// wallet: the transactions are in their graph, the anchors with them, and which transactions +/// paid a script is what their spk index is for. So none of it is persisted alongside +/// [`Subscriptions`], which nothing can reconstruct. +/// +/// Starting empty is always correct, only expensive: a job asks the server for whatever it +/// cannot find here, so an empty one re-downloads every transaction and reproves every anchor. +/// It is not a mirror of the wallet, though — whatever a job fetches lands here too, so it +/// answers "do we already have this" whoever supplied it. +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub struct TxCache { + /// Every txid ever seen for each script hash. + /// + /// This is monotonically growing so that we can detect evictions. + pub spk_txids: HashMap>, + + pub txs: HashMap>, + + /// Written as a sequence: a `(Txid, BlockHash)` key is not a string, so a map would be + /// unserializable in JSON and every other format that requires string keys. + #[serde(with = "persist::anchors_as_seq")] + pub anchors: HashMap<(Txid, BlockHash), ConfirmationBlockTime>, +} + +impl Cache { + pub fn resolve_headers_query( + &mut self, + req: request::Headers, + resp: response::HeadersResp, + ) -> impl Iterator { + self.headers + .extend(resp.headers.iter().map(|&h| (h.block_hash(), h))); + (req.start_height..).zip(resp.headers) + } + + pub fn resolve_history_query( + &mut self, + req: request::GetHistory, + resp: Vec, + ) -> Option { + let status_opt = ElectrumScriptStatus::from_history(&resp); + if let Some(status) = status_opt { + self.tx_cache + .spk_txids + .entry(req.script_hash) + .or_default() + .extend(resp.iter().map(|tx| tx.txid())); + self.subscriptions.insert_spk(req.script_hash, status, resp); + } else { + self.subscriptions.remove_spk(req.script_hash); + } + status_opt + } +} + +/// The last history the server reported for each script hash. +/// +/// Unlike [`TxCache`], a caller cannot rebuild this from wallet data: a status is a hash Electrum +/// computes over the history it stands for and no wallet stores, and the server reports a history +/// as it stands now, never again mentioning a transaction it has dropped. +/// +/// Fields are private: a status is a hash of the history it stands for, and letting the two be +/// set independently would reintroduce the desync the type exists to prevent. +#[derive(Debug, Clone, Default)] +pub struct Subscriptions { + /// The last reported status for a given script. + spk_hash_to_status: HashMap, + /// Script history by status. + /// + /// An entry is dropped once no script answers to its status, which costs a scan of + /// `spk_hash_to_status` on every insert. Fine for a wallet's worth of scripts; a refcount + /// would be the fix if that ever stops being true. + spk_status_to_history: HashMap>, +} + +impl Subscriptions { + fn clear_history_if_no_longer_needed(&mut self, old_status: ElectrumScriptStatus) { + // A status is a hash of the history it stands for, so two scripts paid by the same + // transactions and nothing else share one. The history is only dead once no script + // answers to it any more — dropping it while another still does would leave that + // script with no history and no notification coming to rebuild it. + let still_wanted = self + .spk_hash_to_status + .values() + .any(|&status| status == old_status); + if !still_wanted { + self.spk_status_to_history.remove(&old_status); + } + } + + /// Drop the history for `spk_hash`, for when the server stops reporting one. + pub fn remove_spk(&mut self, spk_hash: ElectrumScriptHash) { + if let Some(old_status) = self.spk_hash_to_status.remove(&spk_hash) { + self.clear_history_if_no_longer_needed(old_status); + } + } + + /// Record `history` as the answer to `spk_status`, replacing whatever `spk_hash` had before. + pub fn insert_spk( + &mut self, + spk_hash: ElectrumScriptHash, + spk_status: ElectrumScriptStatus, + history: Vec, + ) { + if let Some(old_status) = self.spk_hash_to_status.insert(spk_hash, spk_status) { + self.clear_history_if_no_longer_needed(old_status); + } + self.spk_status_to_history.insert(spk_status, history); + } + + /// The history for `spk_hash`, but only if it is the one `spk_status` stands for. + /// + /// A job fetches for the status its notification carried, so handing it a history that + /// answers an older status would have it finish on stale data. `None` sends it to the + /// server instead, which is why the status is effectively part of the key. + pub fn spk_history(&self, spk_status: ElectrumScriptStatus) -> Option<&[response::Tx]> { + self.spk_status_to_history + .get(&spk_status) + .map(Vec::as_slice) + } + + pub fn spk_histories<'a>( + &'a self, + spk_status: impl IntoIterator + 'a, + ) -> impl Iterator + 'a { + spk_status + .into_iter() + .filter_map(|spk_status| self.spk_history(spk_status)) + .flatten() + .filter({ + // Two scripts in the same transaction would otherwise yield it twice. + let mut dedup = BTreeSet::new(); + move |tx: &&response::Tx| dedup.insert(tx.txid()) + }) + .cloned() + } + + /// The last status the server reported for `spk_hash`, if it still has a history. + pub fn spk_status(&self, spk_hash: ElectrumScriptHash) -> Option { + self.spk_hash_to_status.get(&spk_hash).copied() + } + + /// The last status reported for every script that still has a history. + /// + /// This, rather than whichever jobs are currently in flight, is the set of scripts an + /// update has to anchor: a reorg moves transactions the server will never mention again, + /// because one that keeps its height keeps its status. + pub fn spk_statuses(&self) -> impl Iterator + '_ { + self.spk_hash_to_status.values().copied() + } +} + +/// Types and impls that exist only so [`Cache`] can be stored and loaded. +/// +/// Kept apart from the cache itself because [`HistoryTx`] mirrors [`response::Tx`] and the two +/// are easy to mistake for each other at a glance. +mod persist { + use super::*; + + /// A history entry in the shape we can write back out. + /// + /// [`response::Tx`] derives `Deserialize` only, so histories round-trip through this instead. + #[derive(serde::Serialize, serde::Deserialize)] + enum HistoryTx { + Mempool { + txid: Txid, + fee_sats: u64, + confirmed_inputs: bool, + }, + Confirmed { + txid: Txid, + height: u32, + }, + } + + impl From<&response::Tx> for HistoryTx { + fn from(tx: &response::Tx) -> Self { + match tx { + response::Tx::Mempool(tx) => Self::Mempool { + txid: tx.txid, + fee_sats: tx.fee.to_sat(), + confirmed_inputs: tx.confirmed_inputs, + }, + response::Tx::Confirmed(tx) => Self::Confirmed { + txid: tx.txid, + height: tx.height.to_consensus_u32(), + }, + } + } + } + + impl TryFrom for response::Tx { + type Error = bitcoin::absolute::ConversionError; + + fn try_from(tx: HistoryTx) -> Result { + Ok(match tx { + HistoryTx::Mempool { + txid, + fee_sats, + confirmed_inputs, + } => Self::Mempool(response::MempoolTx { + txid, + fee: bitcoin::Amount::from_sat(fee_sats), + confirmed_inputs, + }), + HistoryTx::Confirmed { txid, height } => Self::Confirmed(response::ConfirmedTx { + txid, + height: bitcoin::absolute::Height::from_consensus(height)?, + }), + }) + } + } + + /// Written as `spk_hash -> (status, history)`, which is what rebuilds both maps on the way + /// back in. + impl serde::Serialize for Subscriptions { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_map(self.spk_hash_to_status.iter().map(|(&spk_hash, &status)| { + let history = self + .spk_status_to_history + .get(&status) + .map(|history| history.iter().map(HistoryTx::from).collect::>()) + .unwrap_or_default(); + (spk_hash, (status, history)) + })) + } + } + + impl<'de> serde::Deserialize<'de> for Subscriptions { + fn deserialize>(deserializer: D) -> Result { + use serde::de::Error; + let stored = + HashMap::)>::deserialize( + deserializer, + )?; + let mut spk_histories = Self::default(); + for (spk_hash, (status, history)) in stored { + let history = history + .into_iter() + .map(response::Tx::try_from) + .collect::, _>>() + .map_err(D::Error::custom)?; + spk_histories.insert_spk(spk_hash, status, history); + } + Ok(spk_histories) + } + } + + pub(super) mod anchors_as_seq { + use super::*; + use serde::{Deserialize, Deserializer, Serializer}; + + type Anchors = HashMap<(Txid, BlockHash), ConfirmationBlockTime>; + + pub fn serialize( + anchors: &Anchors, + serializer: S, + ) -> Result { + serializer.collect_seq( + anchors + .iter() + .map(|(&(txid, block_hash), anchor)| (txid, block_hash, anchor)), + ) + } + + pub fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result { + Ok( + Vec::<(Txid, BlockHash, ConfirmationBlockTime)>::deserialize(deserializer)? + .into_iter() + .map(|(txid, block_hash, anchor)| ((txid, block_hash), anchor)) + .collect(), + ) + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use bitcoin::hashes::Hash; + + fn txid(byte: u8) -> Txid { + Txid::from_byte_array([byte; 32]) + } + + fn spk_hash(byte: u8) -> ElectrumScriptHash { + ElectrumScriptHash::from_byte_array([byte; 32]) + } + + /// `response::Tx` derives `Deserialize` only, so histories round-trip through `HistoryTx`. + /// Both of its variants have to survive the trip intact. + #[test] + fn spk_histories_round_trip() { + let history = vec![ + response::Tx::Confirmed(response::ConfirmedTx { + txid: txid(1), + height: bitcoin::absolute::Height::from_consensus(700_000).unwrap(), + }), + response::Tx::Mempool(response::MempoolTx { + txid: txid(2), + fee: bitcoin::Amount::from_sat(1234), + confirmed_inputs: false, + }), + ]; + let status = ElectrumScriptStatus::from_history(&history).expect("history is not empty"); + + let mut before = Subscriptions::default(); + before.insert_spk(spk_hash(9), status, history); + + let json = serde_json::to_string(&before).expect("must serialize"); + let after: Subscriptions = serde_json::from_str(&json).expect("must deserialize"); + + assert_eq!( + after.spk_history(status).map(<[_]>::len), + Some(2), + "the history must survive, and still answer to its status" + ); + assert_eq!(after.spk_status(spk_hash(9)), Some(status)); + } + + /// Two scripts paid by the same transaction, and nothing else, have identical histories — + /// so they share a status. One of them moving on must not take the other's history with it. + #[test] + fn a_shared_history_survives_one_of_its_scripts_moving_on() { + let shared = vec![response::Tx::Confirmed(response::ConfirmedTx { + txid: txid(1), + height: bitcoin::absolute::Height::from_consensus(700_000).unwrap(), + })]; + let shared_status = ElectrumScriptStatus::from_history(&shared).expect("not empty"); + + let mut subs = Subscriptions::default(); + subs.insert_spk(spk_hash(1), shared_status, shared.clone()); + subs.insert_spk(spk_hash(2), shared_status, shared); + + // Script 1 sees another transaction, so its status moves on. Script 2's has not changed, + // and no notification is coming for it. + let moved_on = vec![ + response::Tx::Confirmed(response::ConfirmedTx { + txid: txid(1), + height: bitcoin::absolute::Height::from_consensus(700_000).unwrap(), + }), + response::Tx::Confirmed(response::ConfirmedTx { + txid: txid(2), + height: bitcoin::absolute::Height::from_consensus(700_001).unwrap(), + }), + ]; + let moved_status = ElectrumScriptStatus::from_history(&moved_on).expect("not empty"); + subs.insert_spk(spk_hash(1), moved_status, moved_on); + + assert_eq!(subs.spk_status(spk_hash(2)), Some(shared_status)); + assert!( + subs.spk_history(shared_status).is_some(), + "script 2 still answers to the shared status, so its history must still be there" + ); + } + + /// `anchors` is keyed by a tuple, which JSON cannot use as a map key. A caller who chooses + /// to persist a [`TxCache`] rather than rebuild it must still be able to. + #[test] + fn tx_cache_round_trips_through_json() { + let anchor = (txid(1), bitcoin::BlockHash::from_byte_array([2; 32])); + let mut before = TxCache::default(); + before + .anchors + .insert(anchor, ConfirmationBlockTime::default()); + + let json = serde_json::to_string(&before).expect("must serialize"); + let after: TxCache = serde_json::from_str(&json).expect("must deserialize"); + + assert_eq!(after.anchors.get(&anchor), before.anchors.get(&anchor)); + } + + /// A `Cache` carries none of it, so persisting one cannot go stale against the wallet. + #[test] + fn cache_does_not_persist_the_tx_cache() { + let mut before = Cache::default(); + before.tx_cache.txs.insert( + txid(1), + Arc::new(bitcoin::Transaction { + version: bitcoin::transaction::Version::ONE, + lock_time: bitcoin::absolute::LockTime::ZERO, + input: Vec::new(), + output: Vec::new(), + }), + ); + + let json = serde_json::to_string(&before).expect("must serialize"); + assert!( + !json.contains(&txid(1).to_string()), + "the wallet's own data must not be written here: {json}" + ); + let after: Cache = serde_json::from_str(&json).expect("must deserialize"); + assert!(after.tx_cache.txs.is_empty()); + } +} diff --git a/bdk_electrum_streaming/src/chain_job.rs b/bdk_electrum_streaming/src/chain_job.rs deleted file mode 100644 index e11d156..0000000 --- a/bdk_electrum_streaming/src/chain_job.rs +++ /dev/null @@ -1,129 +0,0 @@ -use crate::req::ReqQueuer; -use bdk_core::{ - bitcoin::{block::Header, BlockHash}, - BlockId, CheckPoint, -}; -use electrum_streaming_client::request; -use std::collections::{BTreeMap, BTreeSet}; - -/// A job that tries to update the [`State`]'s internal [`CheckPoint`] to the latest tip. -/// -/// The job can be completed with [`try_finish()`] given that we have all the blocks required to -/// complete the job. Otherwise, blocks can be introduced to the job with [`process_blocks()`]. -/// -/// [`State`]: crate::State -/// [`try_finish()`]: ChainJob::try_finish -/// [`process_blocks()`]: ChainJob::process_blocks -#[derive(Debug, Clone)] -pub struct ChainJob { - missing_headers: BTreeSet, - cp_update: BTreeMap, -} - -impl ChainJob { - const CHAIN_SUFFIX_LENGTH: u32 = 21; - - /// Construct [`ChainJob`]. - /// - /// Returns `None` if no job is required. I.e. `local_tip` is already at `height` and `header`. - pub fn new( - mut queuer: ReqQueuer, - local_tip: &CheckPoint, - header: Header, - height: u32, - ) -> Option { - let cp = local_tip - .iter() - .find(|cp| cp.height() <= height) - .expect("Local checkpoint must at least have genesis"); - - // Try to short-circuit if possible. - if cp.height() == height { - if cp.hash() == header.block_hash() { - return None; - } - if let Some(prev_cp) = cp.prev() { - if let Some(prev_height) = height.checked_sub(1) { - if prev_height == prev_cp.height() && header.prev_blockhash == prev_cp.hash() { - return Some(Self { - missing_headers: BTreeSet::new(), - cp_update: core::iter::once((height, header.block_hash())).collect(), - }); - } - } - } - } - - let local_start_height = cp.height().saturating_sub(Self::CHAIN_SUFFIX_LENGTH - 1); - let local_height = cp.height(); - let remote_start_height = height.saturating_sub(Self::CHAIN_SUFFIX_LENGTH - 1); - let remote_height = height; - - // Overlap? - if remote_start_height <= local_height { - let start_height = Ord::min(local_start_height, remote_start_height); - let count = (remote_height + 1 - start_height) as usize; - queuer.enqueue(request::Headers { - start_height, - count, - }); - Some(Self { - missing_headers: (start_height..=remote_height).collect(), - cp_update: BTreeMap::new(), - }) - } else { - // Otherwise we have to do two separate requests. - queuer.enqueue(request::Headers { - start_height: local_start_height, - count: (local_height + 1 - local_start_height) as usize, - }); - queuer.enqueue(request::Headers { - start_height: remote_start_height, - count: (remote_height + 1 - remote_start_height) as usize, - }); - Some(Self { - missing_headers: (local_start_height..=local_height) - .chain(remote_start_height..=remote_height) - .collect(), - cp_update: BTreeMap::new(), - }) - } - } - - pub fn process_blocks(mut self, headers: impl IntoIterator) -> Self { - let headers = headers.into_iter().collect::>(); - for (height, header) in headers.iter().cloned() { - if self.missing_headers.remove(&height) { - self.cp_update.insert(height, header); - } - } - tracing::trace!( - processed = headers.len(), - remaining = self.missing_headers.len(), - "Processed blocks for chain job", - ); - self - } - - pub fn try_finish(self, local_tip: &mut CheckPoint) -> Result { - if !self.missing_headers.is_empty() { - tracing::trace!( - missing = self.missing_headers.len(), - "Chain job not finished" - ); - return Err(self); - } - - let mut cp = local_tip.clone(); - for (height, hash) in self.cp_update { - cp = cp.insert(BlockId { height, hash }); - } - *local_tip = cp.clone(); - tracing::info!( - tip_height = cp.height(), - tip_hash = cp.hash().to_string(), - "Chain job finished" - ); - Ok(cp) - } -} diff --git a/bdk_electrum_streaming/src/confirmation_job.rs b/bdk_electrum_streaming/src/confirmation_job.rs new file mode 100644 index 0000000..0c5714f --- /dev/null +++ b/bdk_electrum_streaming/src/confirmation_job.rs @@ -0,0 +1,435 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use bdk_core::{ + bitcoin::{block::Header, BlockHash, Txid}, + BlockId, CheckPoint, +}; +use electrum_streaming_client::{request, ElectrumScriptStatus}; + +use crate::{AnchorUpdate, Cache, ReqQueuer}; + +/// How far along [`ConfirmationJob`] is. +#[derive(Debug, Default, Clone)] +pub enum ConfirmationStage { + #[default] + Init, + FetchBlocks { + to_fetch: BTreeSet, + }, + FetchAnchors { + to_fetch: BTreeSet<(u32, Txid)>, + }, + /// Everything the job set out to get has arrived, and the update has not been taken yet. + /// + /// Kept until [`ConfirmationJob::set_idle`], so a caller not ready to publish can come back + /// for the update on a later poll. + Done, + /// Nothing left to do until the target tip or the statuses move. + /// + /// The update was taken, or the job was abandoned on inconsistent headers. Distinct from + /// [`Done`], which still owes one — a single stage for both would hand the same update over + /// twice, and hand one over for an abandoned job. + /// + /// [`Done`]: Self::Done + Idle, +} + +impl ConfirmationStage { + pub fn fetch_anchors( + cache: &Cache, + spk_statuses: impl IntoIterator, + ) -> Self { + let to_fetch = cache + .subscriptions + .spk_histories(spk_statuses) + .filter_map(|tx| { + let conf_height = tx.confirmation_height()?.to_consensus_u32(); + Some((conf_height, tx.txid())) + }) + .collect(); + Self::FetchAnchors { to_fetch } + } +} + +/// What one [`ConfirmationJob::poll`] achieved. +/// +/// The two `Update` variants are the parts of an [`Update`] this job owns; the rest come from +/// the [`SpkJob`]s, and the caller assembles them. +/// +/// [`Update`]: crate::Update +/// [`SpkJob`]: crate::SpkJob +pub enum ConfirmationProgress { + /// The local chain moved. + CheckPointUpdate { + cp: CheckPoint, + /// If there are any evictions, we need to check which spks need reanchoring + evicted: Vec, + }, + /// Every anchor the job set out to prove, resolved against one chain. + AnchorUpdate(AnchorUpdate), + /// Something changed; poll again. + Continue, + /// Waiting on the server. + Blocked, + /// Finished, and the update is there to be taken. Reported on every poll until it is. + Done, +} + +/// The single job that moves the local chain and anchors what the scripts found. +/// +/// Runs once every [`SpkJob`] has its history — the heights those histories name are all it +/// reads, so a script still downloading its own transactions has already told it every block it +/// needs. Owning the chain and the anchors together is what lets a whole set of anchors be +/// resolved against one chain: resolved per-script, each job raced a tip only this one can +/// move. +/// +/// Responses from an abandoned chain must not reach it. [`Self::set_tip`] reports when the +/// target moved off the chain it was heading for so the caller can forget those requests. +/// +/// [`SpkJob`]: crate::SpkJob +#[derive(Debug, Clone)] +pub struct ConfirmationJob { + target_height: u32, + target_header: Header, + target_statuses: BTreeSet, + + /// Always contains the target header; the notification carries it, so it is never fetched. + fetched_headers: BTreeMap, + stage: ConfirmationStage, +} + +impl ConfirmationJob { + /// An assumption of the max reorg depth. + const MAX_REORG_DEPTH: u32 = 21; + + /// Number of blocks before difficulty adjustment. + const MAX_BATCH_HEADERS_REQUEST: u32 = 2016; + + pub fn new(target_height: u32, target_header: Header) -> Self { + let mut job = ConfirmationJob { + target_height, + target_header, + target_statuses: BTreeSet::default(), + fetched_headers: BTreeMap::default(), + stage: ConfirmationStage::default(), + }; + job.reset_headers(); + job + } + + /// Drop every header but the target's. + /// + /// A tip notification carries the target's header, so it is the one header never worth + /// asking for and the one that must survive any reset — a run that does not link up to it + /// is a chain we were never told about. Reads the target, so set that first. + fn reset_headers(&mut self) { + self.fetched_headers = core::iter::once((self.target_height, self.target_header)).collect(); + } + + pub fn target_tip(&self) -> BlockId { + BlockId { + height: self.target_height, + hash: self.target_header.block_hash(), + } + } + + /// Set the target tip. + /// + /// Returns whether the new tip abandons the chain we were heading for. When it does, every + /// header already in flight answers for that abandoned chain, so the caller must forget + /// those requests before the replacements are queued. + pub fn set_tip(&mut self, height: u32, header: Header) -> bool { + let tip = BlockId { + height, + hash: header.block_hash(), + }; + if self.target_tip() == tip { + return false; + } + let prev = height.checked_sub(1).map(|height| BlockId { + height, + hash: header.prev_blockhash, + }); + // A tip whose parent is the one we were already heading for extends the same chain, so + // the headers gathered for it still describe it. Anything else may not. + let reorged = match prev { + Some(prev) => self.target_tip() != prev, + None => true, + }; + self.target_height = height; + self.target_header = header; + self.stage = ConfirmationStage::Init; + if reorged { + self.reset_headers(); + } else { + self.fetched_headers.insert(height, header); + } + reorged + } + + pub fn set_statuses(&mut self, statuses: impl IntoIterator) { + let statuses = statuses.into_iter().collect::>(); + if self.target_statuses != statuses { + self.target_statuses = statuses; + self.stage = ConfirmationStage::Init; + } + } + + /// Whether the job has finished and its update has not been taken yet. + /// + /// False again after [`Self::set_idle`], so the same update is never handed over twice, and + /// false for a job abandoned mid-fetch. + pub fn is_done(&self) -> bool { + matches!(self.stage, ConfirmationStage::Done) + } + + /// Park the job until the target tip or the statuses move. + /// + /// Call this once the update it was offering has been taken; until then it keeps reporting + /// [`ConfirmationProgress::Done`]. + pub fn set_idle(&mut self) { + self.stage = ConfirmationStage::Idle; + } + + /// Answer the heights the job asked for. + pub fn resolve_blocks(&mut self, blocks: impl IntoIterator) { + self.fetched_headers.extend(blocks); + } + + /// Polls the job as far as it will go. + pub fn poll( + &mut self, + queuer: &mut ReqQueuer, + cache: &Cache, + cp: &CheckPoint, + ) -> anyhow::Result { + match core::mem::take(&mut self.stage) { + ConfirmationStage::Init => { + let to_fetch = self.missing_heights(cache, cp); + + // NOTE: This logic is not perfect and we may duplicate requests due to spk history + // changes between calls to `ConfirmationJob::poll`. Let's not fix it here as we will + // change this crate to download all headers and verify PoW later so there will be + // no need for this logic. + let mut start_height_opt = Option::::None; + let mut iter = to_fetch + .iter() + .copied() + .filter(|h| !self.fetched_headers.contains_key(h)) + .peekable(); + while let Some(h) = iter.next() { + if start_height_opt.is_none() { + start_height_opt = Some(h); + } + let start_height = start_height_opt.expect("must exist"); + if iter.peek().is_some_and(|&next_h| { + next_h <= h.saturating_add(1) + && next_h.saturating_sub(start_height) < Self::MAX_BATCH_HEADERS_REQUEST + }) { + continue; + } + queuer.enqueue(request::Headers { + start_height, + count: (h + 1).saturating_sub(start_height) as usize, + }); + start_height_opt = None; + } + + self.stage = ConfirmationStage::FetchBlocks { to_fetch }; + Ok(ConfirmationProgress::Continue) + } + ConfirmationStage::FetchBlocks { to_fetch } => { + if !to_fetch + .iter() + .all(|h| self.fetched_headers.contains_key(h)) + { + self.stage = ConfirmationStage::FetchBlocks { to_fetch }; + return Ok(ConfirmationProgress::Blocked); + } + + // Headers that disagree mean a reorg landed between fetches; wait to be told. + let mut iter = self + .fetched_headers + .iter() + .rev() + .take((Self::MAX_REORG_DEPTH + 1) as usize) + .peekable(); + while let Some((&height, header)) = iter.next() { + if let Some(&(&prev_height, prev_header)) = iter.peek() { + if prev_height + 1 == height + && prev_header.block_hash() != header.prev_blockhash + { + tracing::info!( + height, + prev_blockhash = header.prev_blockhash.to_string(), + actual_prev_blockhash = prev_header.block_hash().to_string(), + "Fetched headers are inconsistent. Reorg? Abandoning." + ); + self.reset_headers(); + self.stage = ConfirmationStage::Idle; + return Ok(ConfirmationProgress::Blocked); + } + } + } + + // Everything we hold is spliced in from the lowest header up. The target + // header is always one of them, so there is always a run to splice. + let start = self + .fetched_headers + .keys() + .next() + .copied() + .unwrap_or(self.target_height); + let mut extension = BTreeMap::::new(); + let mut base_opt = Option::::None; + for cp in cp.iter() { + if cp.height() < start { + base_opt = Some(cp); + break; + } + extension.insert(cp.height(), cp.hash()); + } + let new_blocks = self + .fetched_headers + .iter() + .map(|(&height, header)| (height, header.block_hash())); + extension.extend(new_blocks); + if extension.get(&0).is_some_and(|&genesis_hash| { + genesis_hash != cp.get(0).expect("genesis must exist").hash() + }) { + return Err(anyhow::anyhow!("server attempted to replace genesis")); + } + let extension = extension + .into_iter() + .map(|(height, hash)| BlockId { height, hash }); + let cp_update = match base_opt { + Some(base) => base.extend(extension).expect("must not error"), + None => CheckPoint::from_block_ids(extension).expect("must not error"), + }; + + let mut evicted_heights = Vec::::new(); + for cp in cp.iter() { + if cp_update + .get(cp.height()) + .is_some_and(|cp_update| cp_update == cp) + { + break; + } + evicted_heights.push(cp.height()); + } + + self.stage = + ConfirmationStage::fetch_anchors(cache, self.target_statuses.iter().copied()); + Ok(ConfirmationProgress::CheckPointUpdate { + cp: cp_update, + evicted: evicted_heights, + }) + } + ConfirmationStage::FetchAnchors { to_fetch } => { + let mut resolved = AnchorUpdate::new(); + let mut all_resolved = true; + for &(height, txid) in &to_fetch { + let header = match self.fetched_headers.get(&height) { + Some(header) => header, + // Not expected to fire: a changed history moves the status set, which + // sends the job back to `Init` to plan this height. Release goes back and + // fetches rather than assume which block this height holds. + None => { + debug_assert!( + false, + "history named height {height}, which the header pass did not cover" + ); + self.stage = ConfirmationStage::Init; + return Ok(ConfirmationProgress::Continue); + } + }; + match cache.tx_cache.anchors.get(&(txid, header.block_hash())) { + Some(&anchor) => { + resolved.insert((anchor, txid)); + } + None => { + all_resolved = false; + queuer.enqueue(request::GetTxMerkle { txid, height }); + } + } + } + if !all_resolved { + // The whole set is kept, not just what is left: each pass resolves all of it + // afresh against the chain as it stands right then, so a reorg landing + // midway cannot leave anchors from two chains in one update. + self.stage = ConfirmationStage::FetchAnchors { to_fetch }; + return Ok(ConfirmationProgress::Blocked); + } + self.stage = ConfirmationStage::Done; + Ok(ConfirmationProgress::AnchorUpdate(resolved)) + } + // `poll` took the stage, so both terminal stages have to put themselves back. + // `Done` still owes an update and keeps offering it until it is taken. + ConfirmationStage::Done => { + self.stage = ConfirmationStage::Done; + Ok(ConfirmationProgress::Done) + } + ConfirmationStage::Idle => { + self.stage = ConfirmationStage::Idle; + Ok(ConfirmationProgress::Blocked) + } + } + } + + /// The heights we still need from the server. + /// + /// Heights whose header is already reachable from `cp` and `cache` are absorbed into + /// `fetched_headers` on the way through, so what comes back is only the gap. + fn missing_heights(&mut self, cache: &Cache, cp: &CheckPoint) -> BTreeSet { + let mut to_fetch = BTreeSet::::new(); + + // Heights the chain itself has to be checked at. Settled first, because a height in + // here is one whose block may be about to be replaced — absorbing it from `cp` below + // would answer the question with the very block under suspicion. + if self.target_tip() != cp.block_id() { + let only_extends_tip = self + .target_height + .checked_sub(1) + .map(|height| { + let hash = self.target_header.prev_blockhash; + BlockId { height, hash } + }) + .is_some_and(|prev| cp.block_id() == prev); + if only_extends_tip { + to_fetch.extend(cp.height() + 1..=self.target_height); + } else { + // Assumes no reorg is deeper than `MAX_REORG_DEPTH`. + let old_tip = cp.height(); + let new_tip = self.target_height; + to_fetch.extend(old_tip.saturating_sub(Self::MAX_REORG_DEPTH)..=old_tip); + to_fetch.extend(new_tip.saturating_sub(Self::MAX_REORG_DEPTH)..=new_tip); + } + } + + // Heights that carry a transaction to anchor. One the chain already places, and whose + // header we have, needs no request. + let anchor_heights = self + .target_statuses + .iter() + .filter_map(|&spk_status| { + let heights = cache + .subscriptions + .spk_history(spk_status)? + .iter() + .filter_map(|tx| Some(tx.confirmation_height()?.to_consensus_u32())); + Some(heights) + }) + .flatten() + .collect::>(); + for height in anchor_heights { + if !to_fetch.insert(height) { + continue; + } + if let Some(&header) = cp.get(height).and_then(|cp| cache.headers.get(&cp.hash())) { + self.fetched_headers.insert(height, header); + } + } + + to_fetch + } +} diff --git a/bdk_electrum_streaming/src/lib.rs b/bdk_electrum_streaming/src/lib.rs index 99ae8df..fbaccc8 100644 --- a/bdk_electrum_streaming/src/lib.rs +++ b/bdk_electrum_streaming/src/lib.rs @@ -1,18 +1,19 @@ //! BDK Electrum goodness. -use bdk_core::spk_client::FullScanResponse; -/// Re-export. +use std::collections::BTreeSet; + +use bdk_core::{bitcoin::Txid, spk_client::FullScanResponse}; pub use electrum_streaming_client; use bdk_core::ConfirmationBlockTime; +mod cache; +pub use cache::*; mod state; use electrum_streaming_client::{ AsyncPendingRequest, BlockingPendingRequest, MaybeBatch, PendingRequest, }; use miniscript::{Descriptor, DescriptorPublicKey}; pub use state::*; -mod chain_job; -pub use chain_job::*; mod req; pub use req::*; mod spk_job; @@ -23,8 +24,11 @@ mod derived_spk_tracker; pub use derived_spk_tracker::*; mod blocking_client; pub use blocking_client::*; +mod confirmation_job; +pub use confirmation_job::*; pub type Update = FullScanResponse; +pub type AnchorUpdate = BTreeSet<(ConfirmationBlockTime, Txid)>; pub type BlockingClientAction = ClientAction>; pub type AsyncClientAction = ClientAction; diff --git a/bdk_electrum_streaming/src/req.rs b/bdk_electrum_streaming/src/req.rs index b1b70db..3577009 100644 --- a/bdk_electrum_streaming/src/req.rs +++ b/bdk_electrum_streaming/src/req.rs @@ -6,7 +6,6 @@ use crate::JobId; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum JobRequest { - GetHeader(request::Header), GetHeaders(request::Headers), GetHistory(request::GetHistory), GetTx(request::GetTx), @@ -25,7 +24,6 @@ pub enum UserRequest { impl JobRequest { pub fn into_raw(self, req_id: u32) -> RawRequest { let (method, params) = match self { - JobRequest::GetHeader(header) => header.to_method_and_params(), JobRequest::GetHeaders(headers) => headers.to_method_and_params(), JobRequest::GetHistory(get_history) => get_history.to_method_and_params(), JobRequest::GetTx(get_tx) => get_tx.to_method_and_params(), @@ -45,12 +43,6 @@ impl JobRequest { } } -impl From for JobRequest { - fn from(value: request::Header) -> Self { - Self::GetHeader(value) - } -} - impl From for JobRequest { fn from(value: request::Headers) -> Self { Self::GetHeaders(value) @@ -87,17 +79,32 @@ impl From for JobRequest { } } +/// A response's originating request, alongside what is needed to route it. +#[derive(Debug, Clone)] +pub struct PoppedRequest { + /// The request this response is answering. + pub request: JobRequest, + /// Jobs awaiting this response. + pub job_ids: BTreeSet, + /// Whether the local chain dropped blocks after this request was sent. + /// + /// The server answers from its own chain, so a response which predates a reorg may describe + /// a block which is no longer the one we have at that height. + pub reorged_since_sent: bool, +} + /// Request coordinator. /// /// Associates responses to their requests and requests to their jobs. #[derive(Debug, Clone, Default)] pub struct ReqCoord { - /// Next request id. next_id: u32, - /// Req id -> Req. - awaiting_responses: HashMap, - /// So we won't have duplicate requests. + /// Req id -> the request, and the chain generation it was enqueued at. + awaiting_responses: HashMap, + /// Also what an identical request is deduplicated against. req_to_job: HashMap>, + /// Bumped every time the local chain drops blocks. + chain_generation: u64, } impl ReqCoord { @@ -112,16 +119,40 @@ impl ReqCoord { &mut self.next_id } - pub fn pop(&mut self, req_id: u32) -> Option<(JobRequest, BTreeSet)> { - let any_req = self.awaiting_responses.remove(&req_id)?; - let job_ids = self.req_to_job.remove(&any_req).unwrap_or_default(); - Some((any_req, job_ids)) + pub fn pop(&mut self, req_id: u32) -> Option { + let (request, generation) = self.awaiting_responses.remove(&req_id)?; + let job_ids = self.req_to_job.remove(&request).unwrap_or_default(); + Some(PoppedRequest { + request, + job_ids, + reorged_since_sent: generation < self.chain_generation, + }) + } + + /// Forget every request `job_id` is still waiting on. + /// + /// A request wanted by another job stays in flight and merely loses `job_id` as an owner. + /// One that nothing else wants is dropped outright, so its response is ignored on arrival + /// and an identical request is no longer deduplicated against it. + pub fn forget_job(&mut self, job_id: JobId) { + let mut orphaned = Vec::new(); + self.req_to_job.retain(|req, job_ids| { + if !job_ids.remove(&job_id) || !job_ids.is_empty() { + return true; + } + orphaned.push(req.clone()); + false + }); + self.awaiting_responses + .retain(|_, (req, _)| !orphaned.contains(req)); } - /// To be called when the network resets. - pub fn clear(&mut self) { - self.awaiting_responses.clear(); - self.req_to_job.clear(); + /// To be called when the local chain drops blocks. + /// + /// Requests already in flight were made against the old chain, so their responses can no + /// longer be trusted to describe the blocks we now have. + pub fn bump_chain_generation(&mut self) { + self.chain_generation += 1; } pub fn queuer<'q>(&'q mut self, queue: &'q mut ReqQueue, job_id: JobId) -> ReqQueuer<'q> { @@ -136,7 +167,7 @@ impl ReqCoord { pub fn pending_requests(&self) -> impl ExactSizeIterator + '_ { self.awaiting_responses .iter() - .map(|(&req_id, req)| req.to_raw(req_id)) + .map(|(&req_id, (req, _))| req.to_raw(req_id)) } } @@ -163,7 +194,10 @@ impl<'q> ReqQueuer<'q> { e.insert(BTreeSet::new()).insert(self.job_id); let req_id = self.coord.next_id; self.coord.next_id = self.coord.next_id.wrapping_add(1); - self.coord.awaiting_responses.insert(req_id, req.clone()); + let generation = self.coord.chain_generation; + self.coord + .awaiting_responses + .insert(req_id, (req.clone(), generation)); self.queue.push_back(req.into_raw(req_id)); } } diff --git a/bdk_electrum_streaming/src/spk_job.rs b/bdk_electrum_streaming/src/spk_job.rs index af45309..0e62b36 100644 --- a/bdk_electrum_streaming/src/spk_job.rs +++ b/bdk_electrum_streaming/src/spk_job.rs @@ -5,76 +5,82 @@ use std::{ use bdk_core::{ bitcoin::{OutPoint, Txid}, - CheckPoint, ConfirmationBlockTime, TxUpdate, + ConfirmationBlockTime, TxUpdate, }; use electrum_streaming_client::{request, response, ElectrumScriptHash, ElectrumScriptStatus}; use crate::{req::ReqQueuer, Cache}; +/// Where a [`SpkJob`] has got to. +/// +/// Each stage names what the job is still waiting for, so being finished is a stage of its own +/// rather than the absence of one. #[derive(Debug)] -pub enum SpkJobStage { - ProcessingHistory { - /// The status for which we are fetching. - status: ElectrumScriptStatus, - }, - ProcessingTxsAndAnchors { - txs: Option, - anchors: BTreeSet<(u32, Txid)>, - }, +pub enum SpkStage { + /// Waiting on the history the status stands for. + ProcessingHistory { status: ElectrumScriptStatus }, + /// Waiting on the transactions that history named. + ProcessingTxs(BTreeSet), + /// Waiting on the outputs those transactions spend. + ProcessingPrevouts(BTreeSet), + /// Everything the job asked for has arrived. + Done, } -impl SpkJobStage { - pub fn done() -> Self { - Self::ProcessingTxsAndAnchors { - txs: None, - anchors: BTreeSet::new(), - } - } - - /// Whether it's done. - pub fn is_done(&self) -> bool { - matches!(self, SpkJobStage::ProcessingTxsAndAnchors { txs, anchors } if txs.is_none() && anchors.is_empty()) - } -} - -#[derive(Debug)] -pub enum TxsJobStage { - Txs(BTreeSet), - Prevouts(BTreeSet), -} - -impl TxsJobStage { - pub fn from_missing_txs(txids: impl IntoIterator) -> Option { +impl SpkStage { + /// What follows a history, given every txid it named. + fn from_txids(txids: impl IntoIterator) -> Self { let txids = txids.into_iter().collect::>(); if txids.is_empty() { - None + Self::Done } else { - Some(Self::Txs(txids)) + Self::ProcessingTxs(txids) } } - pub fn from_missing_prev_txs(outpoints: impl IntoIterator) -> Option { - let prev_txs = outpoints.into_iter().collect::>(); - if prev_txs.is_empty() { - None + /// What follows the transactions, given every output they spend. + fn from_prevouts(outpoints: impl IntoIterator) -> Self { + let prevouts = outpoints.into_iter().collect::>(); + if prevouts.is_empty() { + Self::Done } else { - Some(Self::Prevouts(prev_txs)) + Self::ProcessingPrevouts(prevouts) } } + + pub fn is_done(&self) -> bool { + matches!(self, SpkStage::Done) + } +} + +/// What one [`SpkJob::poll`] achieved. +#[derive(Debug)] +pub enum SpkProgress { + /// A stage completed; poll again. + Continue, + /// Waiting on the server. + Blocked, + /// Everything asked for has arrived. Carries what the job gathered, leaving it empty, so a + /// job polled again after finishing contributes nothing a second time. + Done(TxUpdate), } /// The job to perform once we receive a script status notification. +/// +/// Fetches the script's history, the transactions in it, and the outputs those transactions +/// spend. Anchoring them is [`ConfirmationJob`]'s work: a transaction's anchor depends on the chain, +/// which no single script can move, so resolving anchors per-script had every job racing a +/// tip that only one of them could move. +/// +/// [`ConfirmationJob`]: crate::ConfirmationJob #[derive(Debug)] pub struct SpkJob { - /// Time that we got this notification. + /// When the notification that started this job arrived. pub start: Duration, - /// Script hash of this notification. pub spk_hash: ElectrumScriptHash, - pub stage: SpkJobStage, - - /// Staged tx update. - pub tx_update: TxUpdate, + stage: SpkStage, + tx_update: TxUpdate, } impl SpkJob { @@ -87,14 +93,14 @@ impl SpkJob { let mut tx_update = TxUpdate::default(); let stage = match spk_status { - Some(status) => SpkJobStage::ProcessingHistory { status }, + Some(status) => SpkStage::ProcessingHistory { status }, None => { - if let Some(prev_txids) = cache.spk_txids.get(&spk_hash) { + if let Some(prev_txids) = cache.tx_cache.spk_txids.get(&spk_hash) { tx_update .evicted_ats .extend(prev_txids.iter().map(|&txid| (txid, start.as_secs()))); } - SpkJobStage::done() + SpkStage::Done } }; @@ -106,204 +112,149 @@ impl SpkJob { } } - pub fn elapsed_seconds(&self) -> String { - let duration = UNIX_EPOCH.elapsed().expect("must get current timestamp") - self.start; - let seconds = duration.as_secs(); - let subsec = duration.subsec_millis(); - format!("{seconds}s {subsec}ms") + /// The status this job is still waiting on a history for. + /// + /// `None` once the history is in hand, or when the script had none to begin with. + pub fn awaiting_history(&self) -> Option { + match self.stage { + SpkStage::ProcessingHistory { status } => Some(status), + _ => None, + } } - /// Try fullfill all that is missing. - pub fn advance(mut self, queuer: &mut ReqQueuer, cache: &Cache, cp: &CheckPoint) -> Self { - let mut made_progress = true; - while made_progress { - (self, made_progress) = self.try_advance_once(queuer, cache, cp.clone()); - let stage_str = match &self.stage { - SpkJobStage::ProcessingHistory { status } => format!("ProcessingHistory({status})"), - SpkJobStage::ProcessingTxsAndAnchors { txs, anchors } => { - let inner_str = match txs { - Some(TxsJobStage::Txs(txids)) => format!("txs = {}", txids.len()), - Some(TxsJobStage::Prevouts(ops)) => format!("prevouts = {}", ops.len()), - None => "tx_done".to_string(), - }; - format!( - "ProcessingTxsAndAnchors({inner_str}, anchors = {})", - anchors.len() - ) - } - }; - tracing::trace!( - elapsed_seconds = self.elapsed_seconds(), - spk_hash = self.spk_hash.to_string(), - stage = stage_str, - "Spk job progress" - ); - } - self + /// Whether everything this job asked for has arrived. + pub fn is_done(&self) -> bool { + self.stage.is_done() } - pub fn try_finish(&mut self) -> Option<(ElectrumScriptHash, TxUpdate)> { - if self.stage.is_done() { - tracing::trace!( - elapsed_seconds = self.elapsed_seconds(), - spk_hash = self.spk_hash.to_string(), - "Spk job not finished" - ); - Some((self.spk_hash, core::mem::take(&mut self.tx_update))) - } else { - tracing::info!( - elapsed_seconds = self.elapsed_seconds(), - spk_hash = self.spk_hash.to_string(), - "Spk job finished" - ); - None - } + pub fn elapsed_seconds(&self) -> String { + let now = UNIX_EPOCH.elapsed().expect("must get current timestamp"); + // The system clock can step backwards, which must not bring a log line down with it. + let duration = now.saturating_sub(self.start); + format!("{}s {}ms", duration.as_secs(), duration.subsec_millis()) } - /// Try fullfill all that is missing. + /// Take one step towards having everything the script's history names. /// - /// Returns self + bool representing whether we did advance. - fn try_advance_once( - mut self, - queuer: &mut ReqQueuer, - cache: &Cache, - tip: CheckPoint, - ) -> (Self, bool) { - match self.stage { - SpkJobStage::ProcessingHistory { status } => match cache.spk_histories.get(&status) { - Some(history) => { - if let Some(prev_txids) = cache.spk_txids.get(&self.spk_hash) { - let these_txids = - history.iter().map(|tx| tx.txid()).collect::>(); - let to_evict = prev_txids - .difference(&these_txids) - .map(|&txid| (txid, self.start.as_secs())); - self.tx_update.evicted_ats.extend(to_evict); - } - for tx in history { - if let response::Tx::Mempool(tx) = tx { - self.tx_update - .seen_ats - .insert((tx.txid, self.start.as_secs())); + /// One step per call, so the caller drives it the same way it drives [`ConfirmationJob`]: poll + /// until [`SpkProgress::Blocked`] or [`SpkProgress::Done`]. + /// + /// Errors when the server answers with a transaction that cannot be the one asked for — + /// its outputs do not reach an outpoint we know is spent. That is the server's picture + /// disagreeing with itself, so there is nothing to retry against on this connection. + /// + /// [`ConfirmationJob`]: crate::ConfirmationJob + pub fn poll(&mut self, queuer: &mut ReqQueuer, cache: &Cache) -> anyhow::Result { + let progress = match &mut self.stage { + SpkStage::ProcessingHistory { status } => { + match cache.subscriptions.spk_history(*status) { + Some(history) => { + if let Some(prev_txids) = cache.tx_cache.spk_txids.get(&self.spk_hash) { + let these_txids = + history.iter().map(|tx| tx.txid()).collect::>(); + let to_evict = prev_txids + .difference(&these_txids) + .map(|&txid| (txid, self.start.as_secs())); + self.tx_update.evicted_ats.extend(to_evict); } - } - - let txs = TxsJobStage::from_missing_txs(history.iter().map(|tx| tx.txid())); - let anchors = history - .iter() - .filter_map(|tx| { - let height = tx.confirmation_height()?.to_consensus_u32(); - Some((height, tx.txid())) - }) - .collect(); - self.stage = SpkJobStage::ProcessingTxsAndAnchors { txs, anchors }; - (self, true) - } - None => { - let script_hash = self.spk_hash; - queuer.enqueue(request::GetHistory { script_hash }); - (self, false) - } - }, - SpkJobStage::ProcessingTxsAndAnchors { - mut txs, - mut anchors, - } => { - let mut made_progress = false; - txs = match txs { - Some(TxsJobStage::Txs(mut missing_txs)) => { - missing_txs.retain(|txid| match cache.txs.get(txid) { - Some(tx) => { - self.tx_update.txs.push(tx.clone()); - false - } - None => { - let txid = *txid; - queuer.enqueue(request::GetTx { txid }); - true - } - }); - if missing_txs.is_empty() { - made_progress = true; - TxsJobStage::from_missing_prev_txs( + for tx in history { + if let response::Tx::Mempool(tx) = tx { self.tx_update - .txs - .iter() - .filter(|tx| !tx.is_coinbase()) - .flat_map(|tx| tx.input.iter()) - .map(|txin| txin.previous_output), - ) - } else { - Some(TxsJobStage::Txs(missing_txs)) + .seen_ats + .insert((tx.txid, self.start.as_secs())); + } } + self.stage = SpkStage::from_txids(history.iter().map(|tx| tx.txid())); + SpkProgress::Continue } - Some(TxsJobStage::Prevouts(mut missing_prevouts)) => { - missing_prevouts.retain(|op| match cache.txs.get(&op.txid) { - Some(tx) => { - let txout = match tx.output.get(op.vout as usize) { - Some(txout) => txout, - None => { - debug_assert!(false, "Output must exist in tx"); - unimplemented!("Handle this error"); - } - }; - self.tx_update.txouts.insert(*op, txout.clone()); - false - } - None => { - let txid = op.txid; - queuer.enqueue(request::GetTx { txid }); - true - } + None => { + queuer.enqueue(request::GetHistory { + script_hash: self.spk_hash, }); - if missing_prevouts.is_empty() { - made_progress = true; - None - } else { - Some(TxsJobStage::Prevouts(missing_prevouts)) - } + SpkProgress::Blocked } - None => None, - }; - - let anchors_start_count = anchors.len(); - anchors.retain(|&(height, txid)| { - if height > tip.height() { - // Nothing to request for a block we don't know exists yet. The job is - // re-advanced once a chain job advances the tip. - return true; + } + } + SpkStage::ProcessingTxs(missing_txs) => { + missing_txs.retain(|txid| match cache.tx_cache.txs.get(txid) { + Some(tx) => { + self.tx_update.txs.push(tx.clone()); + false } - - let blockhash = match tip.get(height) { - Some(cp) if cp.height() == height => cp.hash(), - _ => { - queuer.enqueue(request::Header { height }); + None => { + let txid = *txid; + queuer.enqueue(request::GetTx { txid }); + true + } + }); + if missing_txs.is_empty() { + self.stage = SpkStage::from_prevouts( + self.tx_update + .txs + .iter() + .filter(|tx| !tx.is_coinbase()) + .flat_map(|tx| tx.input.iter()) + .map(|txin| txin.previous_output), + ); + SpkProgress::Continue + } else { + SpkProgress::Blocked + } + } + SpkStage::ProcessingPrevouts(missing_prevouts) => { + // `retain` cannot fail, so a bad output is carried out and raised below. + let mut err = Option::::None; + missing_prevouts.retain(|op| { + let tx = match cache.tx_cache.txs.get(&op.txid) { + Some(tx) => tx, + None => { + let txid = op.txid; + queuer.enqueue(request::GetTx { txid }); return true; } }; - - if !cache.headers.contains_key(&blockhash) { - queuer.enqueue(request::Header { height }); - } - - if let Some(anchor) = cache.anchors.get(&(txid, blockhash)) { - self.tx_update.anchors.insert((*anchor, txid)); - return false; - }; - if cache.failed_anchors.contains(&(txid, blockhash)) { - return false; + match tx.output.get(op.vout as usize) { + Some(txout) => { + self.tx_update.txouts.insert(*op, txout.clone()); + } + None => { + err.get_or_insert_with(|| { + anyhow::anyhow!( + "tx {} has {} outputs, but is spent at vout {}", + op.txid, + tx.output.len(), + op.vout, + ) + }); + } } - - queuer.enqueue(request::GetTxMerkle { txid, height }); - true + false }); - if anchors.len() < anchors_start_count { - made_progress = true; + if let Some(err) = err { + return Err(err); + } + if missing_prevouts.is_empty() { + self.stage = SpkStage::Done; + SpkProgress::Continue + } else { + SpkProgress::Blocked } - - self.stage = SpkJobStage::ProcessingTxsAndAnchors { txs, anchors }; - (self, made_progress) } - } + SpkStage::Done => SpkProgress::Done(core::mem::take(&mut self.tx_update)), + }; + + let stage_str = match &self.stage { + SpkStage::ProcessingHistory { status } => format!("ProcessingHistory({status})"), + SpkStage::ProcessingTxs(txids) => format!("ProcessingTxs({})", txids.len()), + SpkStage::ProcessingPrevouts(ops) => format!("ProcessingPrevouts({})", ops.len()), + SpkStage::Done => "Done".to_string(), + }; + tracing::trace!( + elapsed_seconds = self.elapsed_seconds(), + spk_hash = self.spk_hash.to_string(), + stage = stage_str, + "Spk job progress" + ); + Ok(progress) } } diff --git a/bdk_electrum_streaming/src/state.rs b/bdk_electrum_streaming/src/state.rs index 49d3e2e..83d53fe 100644 --- a/bdk_electrum_streaming/src/state.rs +++ b/bdk_electrum_streaming/src/state.rs @@ -1,15 +1,9 @@ -use std::{ - collections::{BTreeMap, BTreeSet, HashMap, HashSet}, - sync::Arc, -}; +use std::collections::BTreeMap; use anyhow::Context; -use bdk_core::{ - bitcoin::{self, BlockHash, Transaction, Txid}, - BlockId, CheckPoint, ConfirmationBlockTime, -}; +use bdk_core::{CheckPoint, ConfirmationBlockTime}; use electrum_streaming_client::{ - notification::Notification, request, response, AsyncPendingRequest, BlockingPendingRequest, + notification::Notification, request, AsyncPendingRequest, BlockingPendingRequest, ElectrumScriptHash, ElectrumScriptStatus, MaybeBatch, PendingRequest, RawNotificationOrResponse, Request, }; @@ -17,16 +11,18 @@ use miniscript::{Descriptor, DescriptorPublicKey}; use serde_json::from_value; use crate::{ - chain_job::ChainJob, - req::{JobRequest, ReqCoord, ReqQueue}, - spk_job::SpkJob, + cache::{Cache, Subscriptions}, + confirmation_job::{ConfirmationJob, ConfirmationProgress}, + req::{JobRequest, PoppedRequest, ReqCoord, ReqQueue}, + spk_job::{SpkJob, SpkProgress}, DerivedSpkTracker, Update, }; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum JobId { Spk(ElectrumScriptHash), - Chain, + /// The single job that moves the local chain and anchors what the scripts found. + Confirmation, } impl JobId { @@ -49,12 +45,17 @@ pub struct State { cache: Cache, spk_jobs: BTreeMap, - chain_job: Option, - user_state: electrum_streaming_client::State, + confirmation_job: Option, - /// Whether we have sent initial requests. + /// The update being built up. /// - /// This includes subscribing to headers, and existing pending requests. + /// Both job kinds write here as they progress, and it is handed to the caller whole when all + /// pending jobs complete. + staged: Update, + + user_state: electrum_streaming_client::State, + + /// Whether the header subscription, spk subscriptions and pending requests have been sent. init_reqs_sent: bool, } @@ -71,24 +72,28 @@ impl State { cp, cache, spk_jobs: BTreeMap::new(), - chain_job: None, + confirmation_job: None, + staged: Update::default(), user_state: electrum_streaming_client::State::new(), init_reqs_sent: false, } } - /// Get a reference to the internal cache. pub fn cache(&self) -> &Cache { &self.cache } + pub fn subscriptions(&self) -> &Subscriptions { + &self.cache.subscriptions + } + /// Reset the state to be not initialized. /// /// Call this after disconnection otherwise pending requests will not be resent and no /// subscriptions to the chain or spks will be made. pub fn reset(&mut self) { tracing::trace!("Reseting state"); - self.chain_job = None; + self.confirmation_job = None; self.init_reqs_sent = false; } @@ -112,13 +117,12 @@ impl State { pub fn init(&mut self, req_queue: &mut ReqQueue) { if !self.init_reqs_sent { self.init_reqs_sent = true; - // Resend pending requests. req_queue.extend(self.user_state.pending_requests()); req_queue.extend(self.coord.pending_requests()); tracing::info!("Queue headers subscribe"); self.coord - .queuer(req_queue, JobId::Chain) + .queuer(req_queue, JobId::Confirmation) .enqueue(request::HeadersSubscribe); for script_hash in self.spk_tracker.all_spk_hashes() { @@ -143,11 +147,41 @@ impl State { ); } - pub fn advance( + pub fn poll( &mut self, req_queue: &mut ReqQueue, raw: RawNotificationOrResponse, ) -> anyhow::Result>> { + self.handle(req_queue, raw)?; + + // Any path through `handle` may have been the one that finished a job, so the update is + // handed over here rather than in each of them. Both job kinds have to be done. + let job = match &mut self.confirmation_job { + Some(job) => job, + None => return Ok(None), + }; + if !job.is_done() || !self.spk_jobs.values().all(SpkJob::is_done) { + return Ok(None); + } + job.set_idle(); + // The scripts have been anchored, so their jobs have served their purpose. + self.spk_jobs.clear(); + let update = core::mem::take(&mut self.staged); + tracing::info!( + tip_height = self.cp.height(), + anchors = update.tx_update.anchors.len(), + txs = update.tx_update.txs.len(), + "Confirmation job finished" + ); + Ok(Some(update)) + } + + /// Apply one message from the server, driving whatever jobs it touches. + fn handle( + &mut self, + req_queue: &mut ReqQueue, + raw: RawNotificationOrResponse, + ) -> anyhow::Result<()> { self.init(req_queue); if let Err(e) = self.user_state.process_incoming(raw.clone()) { match e { @@ -160,84 +194,53 @@ impl State { let notification = Notification::new(&raw_notification) .context("Failed to deserialize notification from server")?; match notification { - Notification::Header(header_notification) => { - // Always replace prev job since a new notification means a new tip. - self.chain_job = ChainJob::new( - self.coord.queuer(req_queue, JobId::Chain), - &self.cp, - *header_notification.header(), - header_notification.height(), - ); - if let Some(job) = self.chain_job.take() { - match job.try_finish(&mut self.cp) { - Ok(cp) => Ok(Some(self.on_chain_job_completed(req_queue, cp))), - Err(job) => { - self.chain_job = Some(job); - Ok(None) - } - } - } else { - Ok(None) - } + Notification::Header(n) => self.on_new_tip(req_queue, n.height(), *n.header()), + Notification::ScriptHash(n) => { + self.on_spk_status(req_queue, n.script_hash(), n.script_status()) } - Notification::ScriptHash(script_hash_notification) => { - let spk_hash = script_hash_notification.script_hash(); - let spk_status = script_hash_notification.script_status(); - - let (k, i) = - self.spk_tracker - .index_of_spk_hash(spk_hash) - .ok_or(anyhow::anyhow!( - "unexpected script hash notification: {}", - spk_hash - ))?; - - let mut last_active_indices = BTreeMap::new(); - - if spk_status.is_some() || self.cache.spk_txids.contains_key(&spk_hash) { - for script_hash in self.spk_tracker.mark_script_hash_used(&k, i) { - self.coord - .queuer(req_queue, JobId::Spk(script_hash)) - .enqueue(request::ScriptHashSubscribe { script_hash }); - } - last_active_indices.insert(k, i); - } - - let mut job = SpkJob::new(&self.cache, spk_hash, spk_status).advance( - &mut self.coord.queuer(req_queue, JobId::Spk(spk_hash)), - &self.cache, - &self.cp, - ); - match job.try_finish() { - Some((_, tx_update)) => { - self.spk_jobs.remove(&spk_hash); - Ok(Some(Update { - tx_update, - last_active_indices, - chain_update: Some(self.cp.clone()), - })) - } - None => { - self.spk_jobs.insert(spk_hash, job); - Ok(None) - } - } - } - Notification::Unknown(_) => Ok(None), + Notification::Unknown(_) => Ok(()), } } RawNotificationOrResponse::Response(raw_response) => { - let (orig_req, job_ids) = match self.coord.pop(raw_response.id) { + let PoppedRequest { + request: orig_req, + job_ids, + reorged_since_sent, + } = match self.coord.pop(raw_response.id) { Some(req) => req, - None => return Ok(None), + None => return Ok(()), }; tracing::trace!(?raw_response, ?orig_req, ?job_ids, "Got raw response"); let raw = match raw_response.result { Ok(raw) => raw, Err(err) => { - // Cancel jobs that resulted in error. - self.cancel_jobs(job_ids); + // Cancel the jobs waiting on this request. + for jid in job_ids { + match jid { + JobId::Spk(spk_hash) => { + self.spk_jobs.remove(&spk_hash); + } + JobId::Confirmation => self.confirmation_job = None, + } + } + + // An anchor fetch is speculative: a reorg may have taken the + // transaction out of the block its height was reported at, or out of the + // chain. An error there is an answer, not a reason to bring the + // connection down. The job went with the loop above, and the next + // notification builds one that asks again. + if let JobRequest::GetTxMerkle(req) = &orig_req { + tracing::warn!( + txid = req.txid.to_string(), + block_height = req.height, + ?err, + "Server gave no merkle proof at this height. Reorg?", + ); + return Ok(()); + } + + // The connection goes down here. return Err(anyhow::anyhow!(err).context("Server responded with error")); } }; @@ -245,94 +248,104 @@ impl State { match orig_req { JobRequest::GetHeaders(req) => { let resp = from_raw(&req, raw)?; - self.cache - .headers - .extend(resp.headers.iter().map(|&h| (h.block_hash(), h))); - debug_assert!(job_ids.contains(&JobId::Chain)); - if let Some(job) = self.chain_job.take() { - let new_blocks = (req.start_height..) - .zip(resp.headers.into_iter().map(|h| h.block_hash())); - match job.process_blocks(new_blocks).try_finish(&mut self.cp) { - Ok(cp) => Ok(Some(self.on_chain_job_completed(req_queue, cp))), - Err(job) => { - self.chain_job = Some(job); - Ok(None) - } - } - } else { - Ok(None) + debug_assert!(job_ids.contains(&JobId::Confirmation)); + let blocks = self + .cache + .resolve_headers_query(req, resp) + .collect::>(); + if let Some(job) = &mut self.confirmation_job { + job.resolve_blocks(blocks); } - } - JobRequest::GetHeader(req) => { - let resp = from_raw(&req, raw)?; - - self.cache - .headers - .insert(resp.header.block_hash(), resp.header); - - // Do not extend checkpoints. - if req.height > self.cp.height() { - return Ok(None); - } - // Do not replace blocks. - if self - .cp - .get(req.height) - .is_some_and(|cp| cp.height() == req.height) - { - return Ok(None); - } - self.cp = self - .cp - .clone() - .insert(BlockId::from((req.height, resp.header.block_hash()))); - Ok(self.advance_spk_jobs(req_queue, job_ids)) + self.poll_confirmation_job(req_queue) } JobRequest::GetHistory(req) => { let resp = from_raw(&req, raw)?; - if let Some(spk_status) = ElectrumScriptStatus::from_history(&resp) { - self.cache - .spk_histories - .entry(spk_status) - .or_default() - .extend(resp.clone()); - self.cache - .spk_txids - .entry(req.script_hash) - .or_default() - .extend(resp.iter().map(|tx| tx.txid())); + let resp_status = self.cache.resolve_history_query(req, resp); + + // A history that does not hash to the status a job awaits can never + // satisfy it, and the two differing means the status moved — so a + // notification is coming, and it will build the job again. + let superseded = self.spk_jobs.get(&req.script_hash).is_some_and(|job| { + job.awaiting_history() + .is_some_and(|awaited| Some(awaited) != resp_status) + }); + if superseded { + tracing::debug!( + spk_hash = req.script_hash.to_string(), + "History answers a status the job is not waiting for. Dropping.", + ); + self.spk_jobs.remove(&req.script_hash); } - Ok(self.advance_spk_jobs(req_queue, job_ids)) + self.poll_spk_jobs(req_queue, job_ids)?; + self.poll_confirmation_job(req_queue) } JobRequest::GetTx(get_tx) => { let resp = from_raw(&get_tx, raw)?; - self.cache.txs.insert(get_tx.txid, resp.tx.into()); - Ok(self.advance_spk_jobs(req_queue, job_ids)) + // The cache is keyed by the txid we asked for, so a transaction that + // hashes to anything else is filed under an id that is not its own, and + // every prevout resolved through it comes from the wrong transaction. + let txid = resp.tx.compute_txid(); + if txid != get_tx.txid { + return Err(anyhow::anyhow!( + "server answered `blockchain.transaction.get` for {} with {}", + get_tx.txid, + txid, + )); + } + self.cache.tx_cache.txs.insert(get_tx.txid, resp.tx.into()); + self.poll_spk_jobs(req_queue, job_ids)?; + self.poll_confirmation_job(req_queue) } JobRequest::GetTxMerkle(req) => { let resp = from_raw(&req, raw)?; + + // The proof answers the server's chain as it was when we asked. Checking + // it against the block we now hold would read a disagreement between two + // chains as a verdict on this one, so discard it and ask again. + if reorged_since_sent { + return self.poll_confirmation_job(req_queue); + } + let cp = match self.cp.get(req.height) { - Some(cp) if cp.height() == req.height => cp, - _ => { - tracing::warn!( + Some(cp) => cp, + // Not expected to fire: the job places every height before it asks + // a proof for it, and a height leaving the chain bumps the generation + // the check above catches. Getting here is our own bookkeeping + // breaking, not the server misbehaving. + None => { + debug_assert!( + false, + "proof for height {}, which is not in our chain", + req.height + ); + tracing::error!( ?req, ?resp, - "Received a merkle proof before we got the header" + "Received a merkle proof before we placed the block" ); - self.cancel_jobs(job_ids); - return Ok(None); + self.confirmation_job = None; + return Ok(()); } }; let header = match self.cache.headers.get(&cp.hash()) { - Some(header) => header, + Some(header) => *header, + // Not expected either, and a reorg is not the reason — that is the + // check above. Every header a job puts in the chain lands in + // `Cache::headers` as it arrives, and nothing prunes them. None => { - tracing::warn!( + debug_assert!( + false, + "no header for {}, the block we hold at height {}", + cp.hash(), + req.height + ); + tracing::error!( ?req, blockhash = cp.hash().to_string(), - "Missing associated header. Reorg?", + "No header for the block we hold at this height", ); - self.cancel_jobs(job_ids); - return Ok(None); + self.confirmation_job = None; + return Ok(()); } }; let exp_root = resp.expected_merkle_root(req.txid); @@ -343,7 +356,7 @@ impl State { block_hash = header.block_hash().to_string(), "Inserting anchor.", ); - self.cache.anchors.insert( + self.cache.tx_cache.anchors.insert( (req.txid, header.block_hash()), ConfirmationBlockTime { block_id: cp.block_id(), @@ -357,143 +370,217 @@ impl State { block_hash = header.block_hash().to_string(), header_root = header.merkle_root.to_string(), expected_root = exp_root.to_string(), - "Failed to verify anchor." + "Proof does not match the block we have at this height", ); - self.cache - .failed_anchors - .insert((req.txid, header.block_hash())); + self.confirmation_job = None; + return Ok(()); } - Ok(self.advance_spk_jobs(req_queue, job_ids)) + self.poll_confirmation_job(req_queue) } JobRequest::ScriptHashSubscribe(req) => { - let spk_hash = req.script_hash; let spk_status = from_raw(&req, raw)?; - - let (k, i) = - self.spk_tracker - .index_of_spk_hash(spk_hash) - .ok_or(anyhow::anyhow!( - "response's request spk was never registered in the spk tracker: {}", - spk_hash - ))?; - - let mut last_active_indices = BTreeMap::new(); - - if spk_status.is_some() || self.cache.spk_txids.contains_key(&spk_hash) { - for script_hash in self.spk_tracker.mark_script_hash_used(&k, i) { - self.coord - .queuer(req_queue, JobId::Spk(script_hash)) - .enqueue(request::ScriptHashSubscribe { script_hash }); - } - last_active_indices.insert(k, i); - } - - let mut job = SpkJob::new(&self.cache, spk_hash, spk_status).advance( - &mut self.coord.queuer(req_queue, JobId::Spk(spk_hash)), - &self.cache, - &self.cp, - ); - - match job.try_finish() { - Some((_, tx_update)) => Ok(Some(Update { - tx_update, - last_active_indices, - chain_update: Some(self.cp.clone()), - })), - None => { - self.spk_jobs.insert(spk_hash, job); - Ok(None) - } - } + self.on_spk_status(req_queue, req.script_hash, spk_status) } JobRequest::HeadersSubscribe(req) => { let resp = from_raw(&req, raw)?; - - // Always replace prev job since a new notification means a new tip. - self.chain_job = ChainJob::new( - self.coord.queuer(req_queue, JobId::Chain), - &self.cp, - resp.header, - resp.height, - ); - if let Some(job) = self.chain_job.take() { - match job.try_finish(&mut self.cp) { - Ok(cp) => Ok(Some(self.on_chain_job_completed(req_queue, cp))), - Err(job) => { - self.chain_job = Some(job); - Ok(None) - } - } - } else { - Ok(None) - } + self.on_new_tip(req_queue, resp.height, resp.header) } } } } } - /// Spk jobs cannot extend the local chain, so a job whose anchor is above the local tip - /// waits with no request in flight. Advancing the tip is what makes such an anchor - /// resolvable, so every completed chain job must re-advance the stashed spk jobs. - fn on_chain_job_completed(&mut self, req_queue: &mut ReqQueue, cp: CheckPoint) -> Update { - let stashed_jobs = self - .spk_jobs - .keys() - .map(|&spk_hash| JobId::Spk(spk_hash)) - .collect::>(); - let mut update = self - .advance_spk_jobs(req_queue, stashed_jobs) - .unwrap_or_default(); - update.chain_update = Some(cp); - update + /// React to the server announcing `header` at `height` as its tip. + fn on_new_tip( + &mut self, + req_queue: &mut ReqQueue, + height: u32, + header: bdk_core::bitcoin::block::Header, + ) -> anyhow::Result<()> { + // A same-height reorg is applied without fetching anything, so this announcement is the + // only place the replacement header is ever offered to us. Caching it here saves the + // anchor refetch a round-trip on the very path it exists for. + self.cache.headers.insert(header.block_hash(), header); + + match &mut self.confirmation_job { + Some(job) => { + if job.set_tip(height, header) { + // The tip we were heading for is gone, and its requests go with it: the + // replacement asks for the same heights, so a survivor would be deduplicated + // against and fill the new job with the chain the server just left. + self.coord.forget_job(JobId::Confirmation); + } + } + None => self.confirmation_job = Some(ConfirmationJob::new(height, header)), + } + self.poll_confirmation_job(req_queue) } - fn advance_spk_jobs( + /// React to the server reporting `spk_status` for `spk_hash`. + fn on_spk_status( + &mut self, + req_queue: &mut ReqQueue, + spk_hash: ElectrumScriptHash, + spk_status: Option, + ) -> anyhow::Result<()> { + let (k, i) = self + .spk_tracker + .index_of_spk_hash(spk_hash) + .ok_or(anyhow::anyhow!( + "unexpected script hash notification: {}", + spk_hash + ))?; + + if spk_status.is_none() { + self.cache.subscriptions.remove_spk(spk_hash); + } + + if spk_status.is_some() || self.cache.tx_cache.spk_txids.contains_key(&spk_hash) { + for script_hash in self.spk_tracker.mark_script_hash_used(&k, i) { + self.coord + .queuer(req_queue, JobId::Spk(script_hash)) + .enqueue(request::ScriptHashSubscribe { script_hash }); + } + self.staged.last_active_indices.insert(k, i); + } + + self.spk_jobs + .insert(spk_hash, SpkJob::new(&self.cache, spk_hash, spk_status)); + self.poll_spk_jobs(req_queue, [JobId::Spk(spk_hash)])?; + // A notification is all that revives a cancelled job, and below the reorg window the + // tip never moves — so this is where an anchor the server has come back to is picked up. + if self.confirmation_job.is_none() { + match self.cache.headers.get(&self.cp.hash()) { + Some(&header) => { + self.confirmation_job = Some(ConfirmationJob::new(self.cp.height(), header)); + } + // Not expected to fire: a tip is only adopted through a notification, which + // caches its header. Ask for the tip rather than leave the anchors waiting on a + // block ten minutes out; a request already in flight absorbs this one. + None => { + tracing::warn!( + tip_height = self.cp.height(), + tip_hash = self.cp.hash().to_string(), + "No header for our tip, so no confirmation job can be built. Resubscribing." + ); + self.coord + .queuer(req_queue, JobId::Confirmation) + .enqueue(request::HeadersSubscribe); + } + } + } + self.poll_confirmation_job(req_queue) + } + + /// Poll the named spk jobs, staging whatever each one has finished gathering. + /// + /// Jobs are left in place when they finish: the update they contribute to is published + /// only once [`ConfirmationJob`] completes, and they are cleared then. + fn poll_spk_jobs( &mut self, req_queue: &mut ReqQueue, job_ids: impl IntoIterator, - ) -> Option> { - let mut update = Option::>::None; - let spk_hashes = job_ids.into_iter().filter_map(|jid| jid.spk_hash()); - for spk_hash in spk_hashes { - if let Some(mut job) = self.spk_jobs.remove(&spk_hash) { - job = job.advance( - &mut self.coord.queuer(req_queue, JobId::Spk(spk_hash)), - &self.cache, - &self.cp, - ); - match job.try_finish() { - Some((spk_hash, tx_update)) => { - let update = update.get_or_insert(Update::default()); - update.tx_update.extend(tx_update); - update + ) -> anyhow::Result<()> { + // Borrowed field by field so a job is polled where it sits, not lifted out and put back. + let Self { + spk_tracker, + coord, + cache, + spk_jobs, + staged, + .. + } = self; + + for spk_hash in job_ids.into_iter().filter_map(JobId::spk_hash) { + let job = match spk_jobs.get_mut(&spk_hash) { + Some(job) => job, + None => continue, + }; + loop { + let mut queuer = coord.queuer(req_queue, JobId::Spk(spk_hash)); + match job.poll(&mut queuer, cache)? { + SpkProgress::Continue => continue, + SpkProgress::Blocked => break, + SpkProgress::Done(tx_update) => { + tracing::info!( + elapsed_seconds = job.elapsed_seconds(), + spk_hash = spk_hash.to_string(), + "Spk job finished" + ); + staged.tx_update.extend(tx_update); + staged .last_active_indices - .extend(self.spk_tracker.index_of_spk_hash(spk_hash)); - } - None => { - self.spk_jobs.insert(spk_hash, job); + .extend(spk_tracker.index_of_spk_hash(spk_hash)); + break; } } } } - if let Some(update) = &mut update { - update.chain_update = Some(self.cp.clone()); - } - update + Ok(()) } - fn cancel_jobs(&mut self, job_ids: impl IntoIterator) { - for jid in job_ids { - match jid { - JobId::Spk(spk_hash) => { - self.spk_jobs.remove(&spk_hash); + /// Drive the confirmation job as far as it will go. + /// + /// Held back only until every script has its history. The job works from the heights those + /// histories name, so a script still downloading the transactions in its own history has + /// already told the job everything it needs — and holding for the downloads would serialise + /// the header and proof fetches behind them for nothing. + fn poll_confirmation_job(&mut self, req_queue: &mut ReqQueue) -> anyhow::Result<()> { + if self + .spk_jobs + .values() + .any(|job| job.awaiting_history().is_some()) + { + return Ok(()); + } + let mut job = match self.confirmation_job.take() { + Some(job) => job, + None => return Ok(()), + }; + // Scoped by what the server has told us about, not by which jobs happen to be live: + // those are cleared on every completion, so a single notification arriving between + // updates would narrow the next reorg's repair to that one script. + job.set_statuses(self.cache.subscriptions.spk_statuses()); + + loop { + let progress = { + let mut queuer = self.coord.queuer(req_queue, JobId::Confirmation); + match job.poll(&mut queuer, &self.cache, &self.cp) { + Ok(progress) => progress, + Err(err) => { + self.confirmation_job = Some(job); + return Err(err); + } + } + }; + match progress { + ConfirmationProgress::Continue => continue, + ConfirmationProgress::CheckPointUpdate { cp, evicted } => { + if !evicted.is_empty() { + tracing::info!( + heights = ?evicted, + "Blocks evicted from the local chain. Refetching anchors." + ); + // Responses to requests which are still in flight describe the chain we + // just left behind. + self.coord.bump_chain_generation(); + } + self.cp = cp.clone(); + self.staged.chain_update = Some(cp); + continue; } - JobId::Chain => { - self.chain_job = None; + ConfirmationProgress::AnchorUpdate(anchors) => { + // Assigned, not extended: each pass re-resolves the whole set at once. + self.staged.tx_update.anchors = anchors; + continue; } + // Nothing more to do this round. Whether the job owes an update is settled by + // `poll`, once the scripts can be checked alongside it. + ConfirmationProgress::Blocked | ConfirmationProgress::Done => break, } } + self.confirmation_job = Some(job); + Ok(()) } } @@ -503,14 +590,3 @@ where { from_value(raw) } - -/// A monotonically growing cache. -#[derive(Debug, Clone, Default)] -pub struct Cache { - pub spk_histories: HashMap>, - pub spk_txids: HashMap>, - pub txs: HashMap>, - pub anchors: HashMap<(Txid, BlockHash), ConfirmationBlockTime>, - pub failed_anchors: HashSet<(Txid, BlockHash)>, - pub headers: HashMap, -} diff --git a/bdk_electrum_streaming/tests/env.rs b/bdk_electrum_streaming/tests/env.rs index 1662d9f..cf23f53 100644 --- a/bdk_electrum_streaming/tests/env.rs +++ b/bdk_electrum_streaming/tests/env.rs @@ -5,7 +5,7 @@ use bdk_chain::{ ChainPosition, IndexedTxGraph, }; use bdk_core::{ - bitcoin::{key::Secp256k1, params::REGTEST, Address, Amount}, + bitcoin::{key::Secp256k1, params::REGTEST, Address, Amount, BlockHash, Txid}, ConfirmationBlockTime, }; use bdk_electrum_streaming::{ @@ -85,11 +85,11 @@ fn blocking_env() -> anyhow::Result<()> { let run_handle = std::thread::spawn(move || { let res = run_blocking( &mut state, - &mut AtomicBool::new(false), + &AtomicBool::new(false), &mut update_tx, &mut client_rx, - &mut &run_conn, - &mut &run_conn, + &run_conn, + &run_conn, ); state.reset(); res @@ -359,3 +359,242 @@ async fn new_block_confirmation_is_anchored_live() -> anyhow::Result<()> { Ok(()) } + +type Graph = IndexedTxGraph>; + +/// The anchor `txid` is canonically confirmed at, if it is confirmed at all. +fn canonical_anchor( + chain: &LocalChain, + graph: &Graph, + txid: Txid, +) -> Option { + graph + .graph() + .list_canonical_txs( + chain, + chain.tip().block_id(), + CanonicalizationParams::default(), + ) + .find(|ctx| ctx.tx_node.txid == txid) + .and_then(|ctx| match ctx.chain_position { + ChainPosition::Confirmed { anchor, .. } => Some(anchor), + ChainPosition::Unconfirmed { .. } => None, + }) +} + +/// A live client against a fresh `electrsd`, with the machinery the reorg tests share. +struct LiveWallet { + env: TestEnv, + chain: LocalChain, + graph: Graph, + update_rx: mpsc::UnboundedReceiver>, + client: AsyncClient<&'static str>, + run_handle: tokio::task::JoinHandle>, +} + +impl LiveWallet { + /// Connect to a fresh test environment and apply the first (genesis) update. + async fn new() -> anyhow::Result { + init(); + + let secp = Secp256k1::new(); + let env = TestEnv::new()?; + let electrum_url = env.electrsd.electrum_url.clone(); + + let (external, _) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[0])?; + let (internal, _) = Descriptor::parse_descriptor(&secp, DESCRIPTORS[1])?; + + let mut graph = IndexedTxGraph::::new({ + let mut indexer = KeychainTxOutIndex::<&'static str>::new(LOOKAHEAD, false); + indexer.insert_descriptor(EXTERNAL, external.clone())?; + indexer.insert_descriptor(INTERNAL, internal.clone())?; + indexer + }); + let (mut chain, _cs) = LocalChain::from_genesis_hash(env.genesis_hash()?); + + let mut spk_tracker = DerivedSpkTracker::<&'static str>::new(LOOKAHEAD); + spk_tracker.insert_descriptor(EXTERNAL, external, 0); + spk_tracker.insert_descriptor(INTERNAL, internal, 0); + + let mut state = AsyncState::new( + ReqCoord::default(), + Cache::default(), + spk_tracker, + chain.tip(), + ); + + let (mut update_tx, mut update_rx) = mpsc::unbounded::>(); + let (client, mut client_rx) = AsyncClient::new(); + + let run_handle = tokio::spawn(async move { + let mut conn = TcpStream::connect(&electrum_url).await?; + let (read, write) = conn.split(); + run_async( + &mut state, + &mut update_tx, + &mut client_rx, + read.compat(), + write.compat_write(), + ) + .await?; + anyhow::Ok(()) + }); + + let update = update_rx.next().await.expect("Must have next update"); + apply_update(&mut chain, &mut graph, update)?; + + Ok(Self { + env, + chain, + graph, + update_rx, + client, + run_handle, + }) + } + + /// Apply updates until `f` holds. + /// + /// Errors if the client stops — which is what a connection torn down by an expected server + /// error looks like from here — or if `f` has not held within the timeout. + async fn wait_until( + &mut self, + what: &str, + mut f: impl FnMut(&LocalChain, &Graph) -> bool, + ) -> anyhow::Result<()> { + let timeout = tokio::time::sleep(Duration::from_secs(150)).fuse(); + pin_mut!(timeout); + loop { + if f(&self.chain, &self.graph) { + return Ok(()); + } + futures::select! { + _ = timeout => return Err(anyhow::anyhow!("timed out waiting for {what}")), + update = self.update_rx.next() => { + let update = update.ok_or_else(|| { + anyhow::anyhow!("the client stopped while waiting for {what}") + })?; + apply_update(&mut self.chain, &mut self.graph, update)?; + }, + } + } + } + + /// Mine past coinbase maturity, then send `Amount::ONE_BTC` to a tracked spk and mine it in. + /// + /// Returns the txid and the hash of the block confirming it. + async fn confirm_tracked_tx(&mut self) -> anyhow::Result<(Txid, BlockHash)> { + self.env.mine_blocks(101, None)?; + let premine_height = self.env.rpc_client().get_block_count()? as u32; + self.wait_until("the premined chain", |chain, _| { + chain.tip().height() >= premine_height + }) + .await?; + + let ((_, spk), _) = self + .graph + .index + .next_unused_spk(EXTERNAL) + .expect("must derive spk"); + let txid = self + .env + .send(&Address::from_script(&spk, ®TEST)?, Amount::ONE_BTC)?; + self.wait_until("the unconfirmed tx", |_, graph| { + graph.graph().get_tx(txid).is_some() + }) + .await?; + + self.env.mine_blocks(1, None)?; + self.wait_until("the tx to be anchored", |chain, graph| { + canonical_anchor(chain, graph, txid).is_some() + }) + .await?; + + let anchor = canonical_anchor(&self.chain, &self.graph, txid).expect("just waited for it"); + Ok((txid, anchor.block_id.hash)) + } + + async fn stop(self) -> anyhow::Result<()> { + self.client.stop().await?; + self.run_handle.await??; + Ok(()) + } +} + +/// Issue #12, end to end: a reorg re-mines a confirmed tx into a *different* block at the *same* +/// height. The Electrum script status is a hash over txid-height pairs, so it is unchanged and no +/// script hash notification is sent. The anchor must still be refetched off the tip alone. +#[tokio::test] +async fn reorg_to_same_height_block_refetches_anchor_live() -> anyhow::Result<()> { + let mut w = LiveWallet::new().await?; + let (txid, first_block) = w.confirm_tracked_tx().await?; + let confirm_height = w.env.rpc_client().get_block_count()? as u32; + + // Invalidate the confirming block and re-mine at the same height. The tx is back in the + // mempool, so it goes into the replacement block too. + w.env.reorg(1)?; + let second_block = w.env.rpc_client().get_best_block_hash()?; + assert_ne!( + first_block, second_block, + "the reorg must actually replace the block" + ); + assert_eq!( + w.env.rpc_client().get_block_count()? as u32, + confirm_height, + "the replacement block must be at the same height" + ); + assert!( + w.env + .rpc_client() + .get_block(&second_block)? + .txdata + .iter() + .any(|tx| tx.compute_txid() == txid), + "the replacement block must still contain the tx" + ); + + w.wait_until("the refetched anchor", |chain, graph| { + canonical_anchor(chain, graph, txid).is_some_and(|a| a.block_id.hash == second_block) + }) + .await?; + + let anchor = canonical_anchor(&w.chain, &w.graph, txid).expect("just waited for it"); + assert_eq!(anchor.block_id.height, confirm_height); + + w.stop().await +} + +/// The everyday reorg: one which takes a tx out of its block and back to the mempool. The anchor +/// refetch then asks for a proof the server cannot give, and that error must not take the +/// connection down with it. +#[tokio::test] +async fn reorg_unconfirming_a_tx_keeps_the_connection_alive() -> anyhow::Result<()> { + let mut w = LiveWallet::new().await?; + let (txid, _) = w.confirm_tracked_tx().await?; + let confirm_height = w.env.rpc_client().get_block_count()? as u32; + + // Invalidate the confirming block and replace it with empty ones, so the tx cannot be + // re-mined and the server has no proof to give at that height. + w.env.invalidate_blocks(1)?; + w.env.mine_empty_block()?; + w.env.mine_empty_block()?; + let tip_height = w.env.rpc_client().get_block_count()? as u32; + assert_eq!(tip_height, confirm_height + 1); + assert!( + w.env.rpc_client().get_raw_mempool()?.contains(&txid), + "the tx must be back in the mempool, so the server really has no proof for it" + ); + + // The connection has to keep serving: `wait_until` fails if the client stops. + w.wait_until("the chain tip after the reorg", |chain, _| { + chain.tip().height() >= tip_height + }) + .await?; + + assert!( + canonical_anchor(&w.chain, &w.graph, txid).is_none(), + "the tx must no longer be canonically confirmed" + ); + + w.stop().await +} diff --git a/bdk_electrum_streaming/tests/state.rs b/bdk_electrum_streaming/tests/state.rs index f95fb27..71c5089 100644 --- a/bdk_electrum_streaming/tests/state.rs +++ b/bdk_electrum_streaming/tests/state.rs @@ -2,9 +2,12 @@ use std::{str::FromStr, sync::Arc}; use bdk_core::{ bitcoin::{ - absolute, block, consensus::encode::serialize_hex, constants, hashes::Hash, transaction, - Amount, CompactTarget, Network, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, - TxMerkleNode, TxOut, Txid, Witness, + absolute, block, + consensus::encode::serialize_hex, + constants, + hashes::{sha256d, Hash}, + transaction, Amount, CompactTarget, Network, OutPoint, ScriptBuf, Sequence, Transaction, + TxIn, TxMerkleNode, TxOut, Txid, Witness, }, BlockId, CheckPoint, ConfirmationBlockTime, }; @@ -23,10 +26,16 @@ fn raw_msg(v: serde_json::Value) -> RawNotificationOrResponse { serde_json::from_value(v).expect("must deserialize raw message") } +#[derive(Clone)] struct Server { headers: Vec, - spk_hash: ElectrumScriptHash, - tx: Transaction, + /// Every transaction the server knows, and the height each is confirmed at. + /// + /// A transaction belongs to the history of whichever scripts its outputs pay, so a server + /// can serve any number of scripts without being told which. + txs: Vec<(Transaction, u32)>, + /// The merkle branch and position this server answers every merkle request with. + merkle_proof: (Vec, usize), } impl Server { @@ -34,8 +43,30 @@ impl Server { self.headers.len() - 1 } - fn answer(&self, req: &RawRequest) -> serde_json::Value { - match req.method.as_ref() { + /// The history of `spk_hash`: every tx paying it that the chain is long enough to contain. + fn history(&self, spk_hash: &serde_json::Value) -> Vec { + self.txs + .iter() + .filter(|(tx, height)| { + self.tip_height() >= *height as usize + && tx.output.iter().any(|txout| { + *spk_hash + == json!(ElectrumScriptHash::new(&txout.script_pubkey).to_string()) + }) + }) + .map(|(tx, height)| { + response::Tx::Confirmed(response::ConfirmedTx { + txid: tx.compute_txid(), + height: absolute::Height::from_consensus(*height) + .expect("must be a valid height"), + }) + }) + .collect() + } + + /// Answer a request, or fail it the way a server would. + fn answer(&self, req: &RawRequest) -> Result { + Ok(match req.method.as_ref() { "blockchain.headers.subscribe" => { let tip = self.headers.last().expect("server must have blocks"); json!({ "hex": serialize_hex(tip), "height": self.tip_height() }) @@ -49,20 +80,58 @@ impl Server { .collect::(); json!({ "count": count, "hex": hex, "max": 2016 }) } - "blockchain.scripthash.subscribe" => json!(null), - "blockchain.scripthash.get_history" => { - if req.params[0] == json!(self.spk_hash.to_string()) && self.tip_height() >= 2 { - json!([{ "tx_hash": self.tx.compute_txid().to_string(), "height": 2 }]) - } else { - json!([]) + "blockchain.block.header" => { + let height = req.params[0].as_u64().expect("must have height") as usize; + match self.headers.get(height) { + Some(header) => json!(serialize_hex(header)), + None => return Err(format!("height {height} is above the chain tip")), + } + } + "blockchain.scripthash.subscribe" => { + match ElectrumScriptStatus::from_history(&self.history(&req.params[0])) { + Some(status) => json!(status.to_string()), + None => json!(null), } } - "blockchain.transaction.get" => json!(serialize_hex(&self.tx)), + "blockchain.scripthash.get_history" => json!(self + .history(&req.params[0]) + .iter() + .map(|tx| json!({ + "tx_hash": tx.txid().to_string(), + "height": tx.electrum_height(), + })) + .collect::>()), + "blockchain.transaction.get" => { + let (tx, _) = self + .txs + .iter() + .find(|(tx, _)| req.params[0] == json!(tx.compute_txid().to_string())) + .expect("must be a tx the server knows"); + json!(serialize_hex(tx)) + } "blockchain.transaction.get_merkle" => { - json!({ "block_height": req.params[1], "merkle": [], "pos": 0 }) + // A proof can only be given for a tx the server has in that block. Both + // romanz/electrs and ElectrumX raise an error otherwise. + let height = req.params[1].as_u64().expect("must have height") as u32; + if !self + .txs + .iter() + .any(|(tx, h)| *h == height && req.params[0] == json!(tx.compute_txid())) + { + return Err(format!( + "tx {} not in block at height {height}", + req.params[0] + )); + } + let (branch, pos) = &self.merkle_proof; + json!({ + "block_height": req.params[1], + "merkle": branch.iter().map(|h| h.to_string()).collect::>(), + "pos": pos, + }) } other => panic!("unexpected request: {other}"), - } + }) } } @@ -73,27 +142,71 @@ fn drain_requests( ) -> Vec> { let mut updates = Vec::new(); while let Some(req) = queue.pop_front() { - let resp = - raw_msg(json!({ "jsonrpc": "2.0", "id": req.id, "result": server.answer(&req) })); - if let Some(update) = state.advance(queue, resp).expect("must advance") { + if let Some(update) = state + .poll(queue, response(&req, server)) + .expect("must poll") + { updates.push(update); } } updates } -/// A history response can report a confirmation height above the local tip: on a new block, -/// romanz/electrs notifies the script hash before the header and answers requests in order, so -/// the history arrives while the local tip is still one block behind. Such an anchor must be -/// deferred until the tip catches up — not dropped — and must still be delivered without any -/// further notification for that script. -#[test] -fn anchor_above_local_tip_is_deferred_until_tip_catches_up() -> anyhow::Result<()> { +/// Drain like [`drain_requests`], but answer every merkle proof in the queue ahead of everything +/// else in each round. +/// +/// The Electrum protocol carries a request id precisely because responses need not come back in +/// the order they were asked for: romanz/electrs happens to answer in order, Fulcrum processes +/// requests concurrently. Nothing may depend on the ordering. +fn drain_requests_proofs_first( + state: &mut BlockingState, + queue: &mut ReqQueue, + server: &Server, +) -> Vec> { + let mut updates = Vec::new(); + while !queue.is_empty() { + let (proofs, rest): (Vec<_>, Vec<_>) = queue + .drain(..) + .partition(|req| req.method.as_ref() == "blockchain.transaction.get_merkle"); + for req in proofs.into_iter().chain(rest) { + if let Some(update) = state + .poll(queue, response(&req, server)) + .expect("must poll") + { + updates.push(update); + } + } + } + updates +} + +/// The server's answer to `req`, as a raw JSON-RPC result or error message. +fn response(req: &RawRequest, server: &Server) -> RawNotificationOrResponse { + raw_msg(match server.answer(req) { + Ok(result) => json!({ "jsonrpc": "2.0", "id": req.id, "result": result }), + Err(message) => json!({ + "jsonrpc": "2.0", + "id": req.id, + "error": { "code": 1, "message": message }, + }), + }) +} + +/// A descriptor to track, the script hash of its first spk, and a tx paying to that spk. +fn tracked_descriptor() -> anyhow::Result<( + Descriptor, + ElectrumScriptHash, + ScriptBuf, +)> { let descriptor = Descriptor::::from_str(&format!("wpkh({XPUB}/0/*)"))?; let spk = descriptor.at_derivation_index(0)?.script_pubkey(); let spk_hash = ElectrumScriptHash::new(&spk); + Ok((descriptor, spk_hash, spk)) +} - let tx = Transaction { +/// A coinbase paying `sats` to `spk`, so that varying `sats` gives a distinct tx. +fn tx_paying(spk: &ScriptBuf, sats: u64) -> Transaction { + Transaction { version: transaction::Version::ONE, lock_time: absolute::LockTime::ZERO, input: vec![TxIn { @@ -103,12 +216,14 @@ fn anchor_above_local_tip_is_deferred_until_tip_catches_up() -> anyhow::Result<( witness: Witness::new(), }], output: vec![TxOut { - value: Amount::from_sat(50_000), - script_pubkey: spk, + value: Amount::from_sat(sats), + script_pubkey: spk.clone(), }], - }; - let txid = tx.compute_txid(); + } +} +/// The regtest genesis block and an empty block on top of it. +fn base_headers() -> (block::Header, block::Header) { let genesis = constants::genesis_block(Network::Regtest).header; let header_1 = block::Header { version: block::Version::ONE, @@ -118,34 +233,89 @@ fn anchor_above_local_tip_is_deferred_until_tip_catches_up() -> anyhow::Result<( bits: CompactTarget::from_consensus(0x207fffff), nonce: 0, }; - // With the tx at position 0 of a single-tx block, the merkle root is its txid. - let header_2 = block::Header { - merkle_root: Txid::to_raw_hash(txid).into(), - prev_blockhash: header_1.block_hash(), - time: 200, - ..header_1 - }; + (genesis, header_1) +} - let mut cache = Cache::default(); - cache.txs.insert(txid, Arc::new(tx.clone())); +/// A block whose transactions have the given merkle root. +fn block_with_root( + prev: &block::Header, + merkle_root: TxMerkleNode, + time: u32, + nonce: u32, +) -> block::Header { + block::Header { + version: block::Version::ONE, + prev_blockhash: prev.block_hash(), + merkle_root, + time, + bits: CompactTarget::from_consensus(0x207fffff), + nonce, + } +} - let mut spk_tracker = DerivedSpkTracker::new(0); - spk_tracker.insert_descriptor("external", descriptor, 0); +/// A block whose only transaction is `txid`, so that its merkle root is the txid itself. +fn block_with_tx(prev: &block::Header, txid: Txid, time: u32, nonce: u32) -> block::Header { + block_with_root(prev, Txid::to_raw_hash(txid).into(), time, nonce) +} - let mut state = BlockingState::new( - ReqCoord::default(), +fn new_state( + cache: Cache, + descriptor: Descriptor, + genesis: block::Header, +) -> BlockingState { + new_state_with_cp( cache, - spk_tracker, + descriptor, CheckPoint::new(BlockId { height: 0, hash: genesis.block_hash(), }), - ); + ) +} + +fn new_state_with_cp( + cache: Cache, + descriptor: Descriptor, + cp: CheckPoint, +) -> BlockingState { + let mut spk_tracker = DerivedSpkTracker::new(0); + spk_tracker.insert_descriptor("external", descriptor, 0); + BlockingState::new(ReqCoord::default(), cache, spk_tracker, cp) +} + +/// The anchor a tx confirmed in `header` at `height` must be given. +fn anchor_of(header: &block::Header, height: u32) -> ConfirmationBlockTime { + ConfirmationBlockTime { + block_id: BlockId { + height, + hash: header.block_hash(), + }, + confirmation_time: header.time as u64, + } +} + +/// A history response can report a confirmation height above the local tip: on a new block, +/// romanz/electrs notifies the script hash before the header and answers requests in order, so +/// the history arrives while the local tip is still one block behind. Such an anchor must be +/// deferred until the tip catches up — not dropped — and must still be delivered without any +/// further notification for that script. +#[test] +fn anchor_above_local_tip_is_deferred_until_tip_catches_up() -> anyhow::Result<()> { + let (descriptor, spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + + let mut cache = Cache::default(); + cache.tx_cache.txs.insert(txid, Arc::new(tx.clone())); + + let mut state = new_state(cache, descriptor, genesis); let mut queue = ReqQueue::new(); let mut server = Server { headers: vec![genesis, header_1], - spk_hash, - tx, + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), }; state.init(&mut queue); @@ -165,7 +335,7 @@ fn anchor_above_local_tip_is_deferred_until_tip_catches_up() -> anyhow::Result<( height: absolute::Height::from_consensus(2)?, })]) .expect("history is not empty"); - state.advance( + state.poll( &mut queue, raw_msg(json!({ "jsonrpc": "2.0", @@ -173,7 +343,7 @@ fn anchor_above_local_tip_is_deferred_until_tip_catches_up() -> anyhow::Result<( "params": [spk_hash.to_string(), status.to_string()], })), )?; - state.advance( + state.poll( &mut queue, raw_msg(json!({ "jsonrpc": "2.0", @@ -185,17 +355,11 @@ fn anchor_above_local_tip_is_deferred_until_tip_catches_up() -> anyhow::Result<( server.headers.push(header_2); let updates = drain_requests(&mut state, &mut queue, &server); - let expected_anchor = ConfirmationBlockTime { - block_id: BlockId { - height: 2, - hash: header_2.block_hash(), - }, - confirmation_time: header_2.time as u64, - }; assert!( - updates - .iter() - .any(|u| u.tx_update.anchors.contains(&(expected_anchor, txid))), + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid))), "anchor must be delivered once the tip catches up" ); Ok(()) @@ -207,7 +371,7 @@ fn anchor_above_local_tip_is_deferred_until_tip_catches_up() -> anyhow::Result<( #[test] fn descriptor_inserted_mid_connection_is_subscribed() -> anyhow::Result<()> { let descriptor = Descriptor::::from_str(&format!("wpkh({XPUB}/0/*)"))?; - let spk_hash = ElectrumScriptHash::new(&descriptor.at_derivation_index(0)?.script_pubkey()); + let spk_hash = ElectrumScriptHash::new(descriptor.at_derivation_index(0)?.script_pubkey()); let genesis = constants::genesis_block(Network::Regtest).header; let mut state = BlockingState::new( @@ -234,3 +398,1734 @@ fn descriptor_inserted_mid_connection_is_subscribed() -> anyhow::Result<()> { ); Ok(()) } + +/// A reorg can move a transaction into a different block at the *same* height. An Electrum +/// script status is a hash over txid-height pairs, so it does not change and the server has no +/// reason to send a script hash notification. The anchor we already delivered now points at a +/// block that is no longer in the chain, so it must be refetched off the tip update alone. +#[test] +fn anchor_is_refetched_when_tx_moves_to_another_block_of_same_height() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + // The block that replaces height 2 contains the tx too, hence the identical script status. + let header_2b = block_with_tx(&header_1, txid, 222, 1); + let header_3b = block::Header { + prev_blockhash: header_2b.block_hash(), + time: 300, + ..header_1 + }; + assert_ne!(header_2.block_hash(), header_2b.block_hash()); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + let updates = drain_requests(&mut state, &mut queue, &server); + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid))), + "tx must first be anchored to the original block" + ); + + // Reorg. Only a header notification is sent: the script status is unchanged, so a server + // has no reason to notify the script hash. + server.headers = vec![genesis, header_1, header_2b, header_3b]; + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_3b), "height": 3 }], + })), + )?; + let updates = drain_requests(&mut state, &mut queue, &server); + + assert!( + updates.iter().any(|u| u + .chain_update + .as_ref() + .is_some_and(|cp| cp.block_id() == anchor_of(&header_3b, 3).block_id)), + "chain update must follow the reorg" + ); + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2b, 2), txid))), + "anchor must be refetched for the block that replaced the evicted one" + ); + Ok(()) +} + +/// A reorg can land while an anchor fetch is in flight. The merkle proof we get back was built +/// against the chain the server had when it received the request, so it may not prove inclusion +/// in the block we now have at that height. Verifying it against that block would record a +/// permanent "not in this block" verdict for an anchor which is in fact valid. +#[test] +fn merkle_proof_predating_a_reorg_is_not_taken_as_a_failed_anchor() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + // The block which replaces height 2 contains the tx alongside another one, so the tx keeps + // its height — and with it its script status — but needs a different merkle proof. + let proof_2b = response::TxMerkle { + block_height: absolute::Height::from_consensus(2)?, + merkle: vec![sha256d::Hash::hash(b"the other tx")], + pos: 1, + }; + let header_2b = block_with_root(&header_1, proof_2b.expected_merkle_root(txid), 222, 1); + let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + // Sync, but hold back the merkle proof so the anchor fetch is still in flight. + state.init(&mut queue); + let mut in_flight = Vec::new(); + while let Some(req) = queue.pop_front() { + if req.method.as_ref() == "blockchain.transaction.get_merkle" { + in_flight.push(req); + continue; + } + state.poll(&mut queue, response(&req, &server))?; + } + let stale_req = match in_flight.as_slice() { + [req] => req.clone(), + reqs => panic!( + "expected exactly one merkle request in flight, got {}", + reqs.len() + ), + }; + let stale_resp = response(&stale_req, &server); + + // The reorg lands before the server answers. + server.headers = vec![genesis, header_1, header_2b, header_3b]; + server.merkle_proof = (proof_2b.merkle.clone(), proof_2b.pos); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_3b), "height": 3 }], + })), + )?; + let mut updates = drain_requests(&mut state, &mut queue, &server); + + // The held answer proves inclusion in the block which was evicted, not in the one which + // replaced it. + updates.extend(state.poll(&mut queue, stale_resp)?); + updates.extend(drain_requests(&mut state, &mut queue, &server)); + + let anchors = updates + .iter() + .flat_map(|u| u.tx_update.anchors.iter().copied()) + .collect::>(); + assert!( + anchors.contains(&(anchor_of(&header_2b, 2), txid)), + "the anchor must be refetched rather than written off from a proof of the evicted block" + ); + Ok(()) +} + +/// A reorg can land while a job is midway through fetching anchors. Anchors are staged as they +/// resolve, so the job must give up the ones it staged against the chain it started on — otherwise +/// it goes on to emit them in a single update alongside the ones it resolved against the chain it +/// ended on. +#[test] +fn anchors_staged_before_a_reorg_are_not_emitted_after_it() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let (tx_a, tx_b) = (tx_paying(&spk, 50_000), tx_paying(&spk, 60_000)); + let (txid_a, txid_b) = (tx_a.compute_txid(), tx_b.compute_txid()); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid_a, 200, 0); + let header_3 = block_with_tx(&header_2, txid_b, 300, 0); + // The reorg keeps both txs at their heights — hence the unchanged script status — but in + // different blocks, and extends the chain by one. + let header_2b = block_with_tx(&header_1, txid_a, 222, 1); + let header_3b = block_with_tx(&header_2b, txid_b, 333, 1); + let header_4b = block_with_root(&header_3b, TxMerkleNode::all_zeros(), 400, 0); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2, header_3], + txs: vec![(tx_a, 2), (tx_b, 3)], + merkle_proof: (Vec::new(), 0), + }; + + // Sync, but hold back tx_b's proof so that the job has staged tx_a's anchor and is still + // waiting on tx_b's when the reorg lands. + state.init(&mut queue); + let mut in_flight = Vec::new(); + while let Some(req) = queue.pop_front() { + if req.method.as_ref() == "blockchain.transaction.get_merkle" + && req.params[0] == json!(txid_b.to_string()) + { + in_flight.push(req); + continue; + } + state.poll(&mut queue, response(&req, &server))?; + } + let held_req = match in_flight.as_slice() { + [req] => req.clone(), + reqs => panic!("expected one held merkle request, got {}", reqs.len()), + }; + let held_resp = response(&held_req, &server); + + server.headers = vec![genesis, header_1, header_2b, header_3b, header_4b]; + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_4b), "height": 4 }], + })), + )?; + let mut updates = drain_requests(&mut state, &mut queue, &server); + updates.extend(state.poll(&mut queue, held_resp)?); + updates.extend(drain_requests(&mut state, &mut queue, &server)); + + let anchors = updates + .iter() + .flat_map(|u| u.tx_update.anchors.iter().copied()) + .collect::>(); + for evicted in [ + (anchor_of(&header_2, 2), txid_a), + (anchor_of(&header_3, 3), txid_b), + ] { + assert!( + !anchors.contains(&evicted), + "an anchor to an evicted block must not be emitted after the reorg: {evicted:?}" + ); + } + for expected in [ + (anchor_of(&header_2b, 2), txid_a), + (anchor_of(&header_3b, 3), txid_b), + ] { + assert!( + anchors.contains(&expected), + "both anchors must be refetched against the new chain: {expected:?}" + ); + } + Ok(()) +} + +/// The everyday reorg: one which takes a transaction out of its block and back to the mempool. +/// The refetch is speculative — we ask for a proof of inclusion at a height the transaction was +/// *last seen* at — so the server answering "not in that block" is expected, and must not take +/// the connection down with it. +#[test] +fn a_tx_unconfirmed_by_a_reorg_does_not_error_the_connection() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + // Height 2 is replaced by a block without the tx, and the chain grows by one. + let header_2b = block_with_root(&header_1, TxMerkleNode::all_zeros(), 222, 1); + let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 333, 0); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + let updates = drain_requests(&mut state, &mut queue, &server); + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid))), + "tx must first be anchored" + ); + + // The reorg leaves the tx in the mempool, so the server no longer has it in any block. + server.headers = vec![genesis, header_1, header_2b, header_3b]; + server.txs = Vec::new(); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_3b), "height": 3 }], + })), + )?; + + while let Some(req) = queue.pop_front() { + state + .poll(&mut queue, response(&req, &server)) + .map_err(|e| anyhow::anyhow!("{e:#}"))?; + } + Ok(()) +} + +/// The other half of `merkle_proof_predating_a_reorg_is_not_taken_as_a_failed_anchor`: a server +/// answers a proof request from the chain it had when it *received* it, so if the tx was out of +/// its block at that moment it answers with an error — an error about the chain we have since +/// left. Blaming that on whichever block the reorg put at the height would write off an anchor +/// which is in fact valid. +#[test] +fn merkle_error_predating_a_reorg_is_not_taken_as_a_failed_anchor() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + // Height 2 is replaced by a block which contains the tx too, so the anchor is still valid. + let header_2b = block_with_tx(&header_1, txid, 222, 1); + let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + // Sync, but hold back the merkle request so the anchor fetch is still in flight. + state.init(&mut queue); + let mut in_flight = Vec::new(); + while let Some(req) = queue.pop_front() { + if req.method.as_ref() == "blockchain.transaction.get_merkle" { + in_flight.push(req); + continue; + } + state.poll(&mut queue, response(&req, &server))?; + } + let stale_req = match in_flight.as_slice() { + [req] => req.clone(), + reqs => panic!( + "expected exactly one merkle request in flight, got {}", + reqs.len() + ), + }; + + // The reorg lands before the server answers. + server.headers = vec![genesis, header_1, header_2b, header_3b]; + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_3b), "height": 3 }], + })), + )?; + let mut updates = drain_requests(&mut state, &mut queue, &server); + + // The held request was received while the tx was out of its block, so it is answered with + // an error — the wording is the one romanz/electrs really sends, a bare JSON string which + // conflates a genuine fault with the everyday reorg. + updates.extend(state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "id": stale_req.id, + "error": "tx not found or is unconfirmed", + })), + )?); + updates.extend(drain_requests(&mut state, &mut queue, &server)); + + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2b, 2), txid))), + "the anchor must be refetched rather than written off from an error about the evicted block" + ); + Ok(()) +} + +/// A server error is not a disproof — it is equally a rate limit, an index still catching up or +/// a daemon hiccup — so it must not be recorded as one. The transaction stays in the record of +/// what was seen at that height, so the next reorg of that height asks again; and the job it +/// blocked must still finish rather than re-ask in a loop. +#[test] +fn a_merkle_error_is_not_recorded_as_a_failed_anchor() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + // Height 2 is replaced by a block containing the tx, and the chain grows by one. + let header_2b = block_with_tx(&header_1, txid, 222, 1); + let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2], + txs: vec![(tx.clone(), 2)], + merkle_proof: (Vec::new(), 0), + }; + + // Sync, but fail every merkle request the way a busy server would. + state.init(&mut queue); + let mut merkle_requests = 0; + while let Some(req) = queue.pop_front() { + let resp = if req.method.as_ref() == "blockchain.transaction.get_merkle" { + merkle_requests += 1; + assert!( + merkle_requests < 10, + "a server error must not put the job in a re-ask loop" + ); + raw_msg(json!({ + "jsonrpc": "2.0", + "id": req.id, + "error": { "code": 1, "message": "server busy" }, + })) + } else { + response(&req, &server) + }; + state.poll(&mut queue, resp)?; + } + assert_eq!( + merkle_requests, 1, + "the job must give up on the pair rather than re-ask" + ); + assert!( + state.cache().tx_cache.anchors.is_empty(), + "an error proves nothing, so no anchor may be recorded from it" + ); + + // The reorg replaces the block, so the anchor is asked for again — which it could not be + // had the error dropped this script from `spk_hashes_by_height`. + server.headers = vec![genesis, header_1, header_2b, header_3b]; + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_3b), "height": 3 }], + })), + )?; + let updates = drain_requests(&mut state, &mut queue, &server); + + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2b, 2), txid))), + "the tx must still be asked about at this height after a server error" + ); + Ok(()) +} + +/// Issue #12's literal case: a reorg to a block of the *same* height, with no growth at all. +/// The tip announcement carries the replacement header, so this is the one reorg shape that can +/// be applied without fetching a single block — and every other reorg test here also grows the +/// chain, which takes a different path. +#[test] +fn anchor_is_refetched_after_a_same_height_reorg() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + let header_2b = block_with_tx(&header_1, txid, 222, 1); + assert_ne!(header_2.block_hash(), header_2b.block_hash()); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + let updates = drain_requests(&mut state, &mut queue, &server); + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid))), + "tx must first be anchored to the original block" + ); + + // The tip does not move: same height, different block, unchanged script status. + server.headers = vec![genesis, header_1, header_2b]; + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_2b), "height": 2 }], + })), + )?; + let updates = drain_requests(&mut state, &mut queue, &server); + + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2b, 2), txid))), + "anchor must be refetched for the block that replaced the evicted one" + ); + Ok(()) +} + +/// A proof is verified against the merkle root of the block we have at that height, so a job +/// which asks for the proof and that block's header together only works if the server answers in +/// request order. It need not: the protocol carries request ids for that reason. +#[test] +fn anchor_is_refetched_whatever_order_the_server_answers_in() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + // Height 2 is replaced and the chain grows, so the notified header is height 3's — the + // replacement block's header has to be fetched before its proof can be verified. + let header_2b = block_with_tx(&header_1, txid, 222, 1); + let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + let updates = drain_requests(&mut state, &mut queue, &server); + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid))), + "tx must first be anchored to the original block" + ); + + server.headers = vec![genesis, header_1, header_2b, header_3b]; + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_3b), "height": 3 }], + })), + )?; + let updates = drain_requests_proofs_first(&mut state, &mut queue, &server); + + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2b, 2), txid))), + "the anchor must be refetched even when the proof overtakes the header" + ); + Ok(()) +} + +/// A client restored from a persisted checkpoint chain starts with blocks in its chain whose +/// headers are not in its cache. Resolving an anchor at such a height needs both the header and +/// the proof, and nothing else will fetch that header — the chain consistency pass has nothing to +/// do, the tip being already correct. +/// +/// So this is the case where the two halves of the ordering fix are load-bearing: the proof must +/// not be asked for before the header is cached, and the header response must poll the waiting +/// job even though the chain itself has nothing to learn from it. +#[test] +fn anchor_resolves_when_the_chain_is_restored_without_its_headers() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + let header_3 = block_with_root(&header_2, TxMerkleNode::all_zeros(), 300, 0); + + // The restored chain knows the blocks, the fresh cache knows none of their headers. The tx + // is one block below the tip, so the header it needs is not the one `headers.subscribe` + // hands back, and the chain consistency pass has nothing to do either, the tip being + // already correct. + let cp = CheckPoint::new(BlockId { + height: 0, + hash: genesis.block_hash(), + }) + .insert(BlockId { + height: 2, + hash: header_2.block_hash(), + }) + .insert(BlockId { + height: 3, + hash: header_3.block_hash(), + }); + let mut state = new_state_with_cp(Cache::default(), descriptor, cp); + let mut queue = ReqQueue::new(); + let server = Server { + headers: vec![genesis, header_1, header_2, header_3], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + let updates = drain_requests_proofs_first(&mut state, &mut queue, &server); + + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid))), + "the anchor must resolve even when the proof overtakes the header it is verified against" + ); + Ok(()) +} + +/// A header batch fetched before a reorg describes the chain we have since left behind, and +/// splicing it in would put a purged block into the checkpoint chain. +/// +/// The case that exposes it is a *sparse* chain — a restored one, or one whose missing heights +/// sit below the reorg window `ConfirmationJob` rewrites — reorged deeper than that window, so the +/// consistency pass never learns the low block changed too. Only the height the anchor needs +/// brings it back, and that fetch was in flight when the chain moved. +#[test] +fn header_fetched_before_a_reorg_is_not_spliced_into_the_chain() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + + // Two chains which differ at height 2 as well as near the tip. Both contain the tx at + // height 2, so the anchor stays valid throughout — only the block it belongs to changes. + let build = |second: block::Header, tip: u32, nonce: u32| { + let mut chain = vec![genesis, header_1, second]; + for height in 3..=tip { + let prev = *chain.last().expect("non-empty"); + chain.push(block_with_root( + &prev, + TxMerkleNode::all_zeros(), + 1000 + height, + nonce, + )); + } + chain + }; + let chain_a = build(block_with_tx(&header_1, txid, 200, 0), 30, 0); + let chain_b = build(block_with_tx(&header_1, txid, 222, 1), 31, 1); + let (a2, b2) = (chain_a[2], chain_b[2]); + assert_ne!(a2.block_hash(), b2.block_hash()); + + // A restored chain sparse enough that height 2 is a gap — so the anchor has to fetch that + // header, and `replaces` will not decline it when it comes back. + let cp = CheckPoint::new(BlockId { + height: 0, + hash: genesis.block_hash(), + }) + .insert(BlockId { + height: 30, + hash: chain_a[30].block_hash(), + }); + let mut state = new_state_with_cp(Cache::default(), descriptor, cp); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: chain_a.clone(), + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + // Sync, but hold back the batch covering height 2 so that fetch is still in flight. + state.init(&mut queue); + let mut in_flight = Vec::new(); + while let Some(req) = queue.pop_front() { + if req.method.as_ref() == "blockchain.block.headers" { + let start = req.params[0].as_u64().expect("must have start_height"); + let count = req.params[1].as_u64().expect("must have count"); + if (start..start + count).contains(&2) { + in_flight.push(req); + continue; + } + } + state.poll(&mut queue, response(&req, &server))?; + } + let stale_req = match in_flight.as_slice() { + [req] => req.clone(), + reqs => panic!("expected one held header batch, got {}", reqs.len()), + }; + let stale_resp = response(&stale_req, &server); + + // The reorg lands. It runs deeper than the reorg window, so the consistency pass rewrites + // only the top of the chain and never learns that height 2 changed too. + server.headers = chain_b.clone(); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&chain_b[31]), "height": 31 }], + })), + )?; + let mut updates = drain_requests(&mut state, &mut queue, &server); + + // The held answer describes the chain we have left behind. + updates.extend(state.poll(&mut queue, stale_resp)?); + updates.extend(drain_requests(&mut state, &mut queue, &server)); + + let tip = updates + .iter() + .rev() + .find_map(|u| u.chain_update.clone()) + .expect("must get a chain update"); + let at_2 = tip.iter().find(|cp| cp.height() == 2); + assert_ne!( + at_2.as_ref().map(|cp| cp.hash()), + Some(a2.block_hash()), + "a header from the chain we left must not be spliced into the checkpoint chain" + ); + assert_eq!( + at_2.map(|cp| cp.hash()), + Some(b2.block_hash()), + "the height must be refetched against the chain we are actually on" + ); + assert!( + updates + .iter() + .any(|u| u.tx_update.anchors.contains(&(anchor_of(&b2, 2), txid))), + "and the anchor must resolve against that block" + ); + Ok(()) +} + +/// The refetch is a script hash notification we raise ourselves, so it must not displace one the +/// server actually sent. A real notification carries a status at least as new as anything we +/// could replay from cache; replacing its job would resolve the script against a stale history +/// and drop whatever the new status was reporting. +#[test] +fn a_replayed_job_does_not_displace_one_the_server_started() -> anyhow::Result<()> { + let (descriptor, spk_hash, spk) = tracked_descriptor()?; + let (tx_a, tx_b) = (tx_paying(&spk, 50_000), tx_paying(&spk, 60_000)); + let (txid_a, txid_b) = (tx_a.compute_txid(), tx_b.compute_txid()); + let (genesis, _) = base_headers(); + let header_1 = block_with_tx(&genesis, txid_b, 100, 0); + let header_2 = block_with_tx(&header_1, txid_a, 200, 0); + // The reorg replaces height 2 with another block holding tx_a, and extends the chain. + let header_2b = block_with_tx(&header_1, txid_a, 222, 1); + let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2], + // The server only reports tx_a to begin with. + txs: vec![(tx_a, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + let updates = drain_requests(&mut state, &mut queue, &server); + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid_a))), + "tx_a must first be anchored to the original block" + ); + + // The server now reports tx_b as well, and notifies the new status. The job that starts is + // the only thing which knows about tx_b. + server.txs.insert(0, (tx_b, 1)); + let new_status = + ElectrumScriptStatus::from_history(&server.history(&json!(spk_hash.to_string()))) + .expect("history must be non-empty"); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.scripthash.subscribe", + "params": [spk_hash.to_string(), new_status.to_string()], + })), + )?; + + // Hold back that job's history, so it is still in flight when the reorg lands. + let mut in_flight = Vec::new(); + while let Some(req) = queue.pop_front() { + if req.method.as_ref() == "blockchain.scripthash.get_history" { + in_flight.push(req); + continue; + } + state.poll(&mut queue, response(&req, &server))?; + } + let held_req = match in_flight.as_slice() { + [req] => req.clone(), + reqs => panic!("expected one held history request, got {}", reqs.len()), + }; + + // The reorg evicts height 2, where this script is recorded — so the refetch wants to replay + // its job, and must decline because the server's own job is already there. + server.headers = vec![genesis, header_1, header_2b, header_3b]; + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_3b), "height": 3 }], + })), + )?; + let mut updates = drain_requests(&mut state, &mut queue, &server); + updates.extend(state.poll(&mut queue, response(&held_req, &server))?); + updates.extend(drain_requests(&mut state, &mut queue, &server)); + + assert!( + updates + .iter() + .any(|u| u.tx_update.txs.iter().any(|tx| tx.compute_txid() == txid_b)), + "the server's own job must survive and deliver what its status was reporting" + ); + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2b, 2), txid_a))), + "and must still refetch the anchor the reorg invalidated" + ); + Ok(()) +} + +/// A persisted checkpoint chain can be stale at a height below the window `ConfirmationJob` rewrites +/// — an offline reorg, say. Then the block *we* have at that height is one the server does not +/// have, and a header request is keyed by height, so no request can ever fetch it. +/// +/// Withholding the proof until that header is cached must not turn into an endless request loop. +#[test] +fn a_header_the_server_does_not_have_does_not_loop() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + // The server's height 2, and the stale one our persisted chain still claims. + let server_2 = block_with_tx(&header_1, txid, 200, 0); + let stale_2 = block_with_tx(&header_1, txid, 999, 7); + assert_ne!(server_2.block_hash(), stale_2.block_hash()); + + let mut chain = vec![genesis, header_1, server_2]; + for height in 3..=30u32 { + let prev = *chain.last().expect("non-empty"); + chain.push(block_with_root( + &prev, + TxMerkleNode::all_zeros(), + 1000 + height, + 0, + )); + } + + // Tip agrees with the server, so the consistency pass never rewrites height 2. + let cp = CheckPoint::new(BlockId { + height: 0, + hash: genesis.block_hash(), + }) + .insert(BlockId { + height: 2, + hash: stale_2.block_hash(), + }) + .insert(BlockId { + height: 30, + hash: chain[30].block_hash(), + }); + let mut state = new_state_with_cp(Cache::default(), descriptor, cp); + let mut queue = ReqQueue::new(); + let server = Server { + headers: chain, + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + let mut served = 0; + while let Some(req) = queue.pop_front() { + served += 1; + assert!( + served < 200, + "the client must not loop: {served} requests, last was {} {:?}", + req.method, + req.params + ); + state.poll(&mut queue, response(&req, &server))?; + } + Ok(()) +} + +/// A history that comes back empty is the server saying the script has nothing, which is as much +/// an answer as a null status in a notification — and the two paths have to agree, or a script +/// whose transactions vanished between the notification and the answer keeps a status no script +/// should still answer to, and the confirmation job goes on anchoring transactions the server no +/// longer lists. +/// +/// The history behind that status has to go too. It is keyed by status, not by script, so leaving +/// it once nothing points at it strands a `Vec` in a structure that is persisted. +#[test] +fn a_history_that_comes_back_empty_clears_the_subscription() -> anyhow::Result<()> { + let (descriptor, spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + drain_requests(&mut state, &mut queue, &server); + let old_status = state + .subscriptions() + .spk_status(spk_hash) + .expect("the script has a history to start with"); + assert!( + state.subscriptions().spk_history(old_status).is_some(), + "the history behind that status must be held" + ); + + // The server reports a second payment, so the status moves and the job goes to fetch the + // history it stands for. + let tx2 = tx_paying(&spk, 60_000); + server.txs.push((tx2, 0)); + let new_status = + ElectrumScriptStatus::from_history(&server.history(&json!(spk_hash.to_string()))) + .expect("history must be non-empty"); + assert_ne!(new_status, old_status); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.scripthash.subscribe", + "params": [spk_hash.to_string(), new_status.to_string()], + })), + )?; + + // By the time we ask, everything the script had is gone — both payments replaced. The server + // answers with an empty history, which no status stands for. + let mut answered_empty = false; + while let Some(req) = queue.pop_front() { + let resp = if req.method.as_ref() == "blockchain.scripthash.get_history" { + answered_empty = true; + raw_msg(json!({ "jsonrpc": "2.0", "id": req.id, "result": [] })) + } else { + response(&req, &server) + }; + state.poll(&mut queue, resp)?; + } + assert!(answered_empty, "the job must have asked for the history"); + + assert!( + state.subscriptions().spk_status(spk_hash).is_none(), + "an empty history must clear the script's status" + ); + assert!( + state.subscriptions().spk_history(old_status).is_none(), + "and drop the history nothing answers to any more" + ); + Ok(()) +} + +/// A replay is built from the last status and the heights it was seen at, so both have to go +/// when the server stops reporting a history for the script — an RBF'd transaction, say. Left +/// behind, a later eviction at that height would rebuild the job from a history the script no +/// longer has and go asking for proofs of a transaction that is gone. +#[test] +fn a_script_whose_history_goes_away_is_not_replayed() -> anyhow::Result<()> { + let (descriptor, spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + let header_2b = block_with_root(&header_1, TxMerkleNode::all_zeros(), 222, 1); + let header_3b = block_with_root(&header_2b, TxMerkleNode::all_zeros(), 300, 0); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + let updates = drain_requests(&mut state, &mut queue, &server); + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid))), + "tx must first be anchored" + ); + assert!( + state.subscriptions().spk_status(spk_hash).is_some(), + "the status must be recorded while the script has a history" + ); + + // The transaction is gone, so the script's status goes to null. + server.txs = Vec::new(); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.scripthash.subscribe", + "params": [spk_hash.to_string(), null], + })), + )?; + drain_requests(&mut state, &mut queue, &server); + assert!( + !state.subscriptions().spk_status(spk_hash).is_some(), + "a null status must drop the recorded status" + ); + + // A reorg evicting the height it used to be seen at must not replay anything. + server.headers = vec![genesis, header_1, header_2b, header_3b]; + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_3b), "height": 3 }], + })), + )?; + let mut asked = Vec::new(); + while let Some(req) = queue.pop_front() { + asked.push(req.method.to_string()); + state.poll(&mut queue, response(&req, &server))?; + } + assert!( + !asked + .iter() + .any(|m| m == "blockchain.transaction.get_merkle"), + "no proof may be asked for a transaction the script no longer has, got: {asked:?}" + ); + Ok(()) +} + +/// A server proves inclusion in whichever block *it* has at a height, so a proof whose root does +/// not match ours says the two chains disagree there — not that the transaction is absent from +/// our block. Remembering that against our block would be a verdict the proof cannot support, and +/// a chain that came back to that block would consult it and skip the anchor for good. +#[test] +fn a_proof_for_another_block_is_not_a_verdict_on_ours() -> anyhow::Result<()> { + let (descriptor, spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let ours = block_with_tx(&header_1, txid, 200, 0); + // Their block holds the tx alongside another, so it needs a different proof — the root the + // server's proof expands to cannot match the root of our block. + let proof_theirs = response::TxMerkle { + block_height: absolute::Height::from_consensus(2)?, + merkle: vec![sha256d::Hash::hash(b"the other tx")], + pos: 1, + }; + let theirs = block_with_root(&header_1, proof_theirs.expected_merkle_root(txid), 222, 1); + assert_ne!(ours.merkle_root, theirs.merkle_root); + + let mut chain = vec![genesis, header_1, theirs]; + for height in 3..=30u32 { + let prev = *chain.last().expect("non-empty"); + chain.push(block_with_root( + &prev, + TxMerkleNode::all_zeros(), + 1000 + height, + 0, + )); + } + + // A persisted chain holding our block at height 2, and its header already cached — otherwise + // `GetHeader` catches the disagreement before any proof is asked for. The tip agrees, so no + // chain job runs and nothing rewrites height 2: the disagreement is below the window. + let mut cache = Cache::default(); + cache.headers.insert(ours.block_hash(), ours); + let cp = CheckPoint::new(BlockId { + height: 0, + hash: genesis.block_hash(), + }) + .insert(BlockId { + height: 2, + hash: ours.block_hash(), + }) + .insert(BlockId { + height: 30, + hash: chain[30].block_hash(), + }); + let mut state = new_state_with_cp(cache, descriptor, cp); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: chain, + txs: vec![(tx, 2)], + merkle_proof: (proof_theirs.merkle.clone(), proof_theirs.pos), + }; + + state.init(&mut queue); + let mut served = 0; + while let Some(req) = queue.pop_front() { + served += 1; + assert!(served < 200, "a mismatch must not become a request loop"); + state.poll(&mut queue, response(&req, &server))?; + } + assert!( + state.cache().tx_cache.anchors.is_empty(), + "a proof for a block we do not have must not anchor anything" + ); + + // The server comes back to our block at that height. Nothing durable was written against it, + // so the job a notification rebuilds must be able to anchor there. + server.headers[2] = ours; + server.merkle_proof = (Vec::new(), 0); + let status = ElectrumScriptStatus::from_history(&server.history(&json!(spk_hash.to_string()))) + .expect("history must be non-empty"); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.scripthash.subscribe", + "params": [spk_hash.to_string(), status.to_string()], + })), + )?; + let updates = drain_requests(&mut state, &mut queue, &server); + assert!( + updates + .iter() + .any(|u| u.tx_update.anchors.contains(&(anchor_of(&ours, 2), txid))), + "the anchor must still be reachable once the chains agree again" + ); + Ok(()) +} + +/// A server that will not prove a transaction at a height says our two chains disagree there. It +/// does not say the transaction is absent from our block, and below the reorg window nothing +/// rewrites that height, so the tip never moves and no tip notification is coming. +/// +/// The script notification is the only thing that comes back, and it can only revive a job that +/// still exists — [`ConfirmationJob`] is built in `on_new_tip` and nowhere else. Dropping the job +/// on the error would strand the anchor until an unrelated block arrives. +#[test] +fn a_merkle_error_below_the_reorg_window_is_recovered_by_a_script_notification( +) -> anyhow::Result<()> { + let (descriptor, spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let ours = block_with_tx(&header_1, txid, 200, 0); + // Their block at that height does not hold the tx, so they will not prove it there. + let theirs = block_with_root(&header_1, TxMerkleNode::all_zeros(), 222, 1); + assert_ne!(ours.merkle_root, theirs.merkle_root); + + let mut chain = vec![genesis, header_1, theirs]; + for height in 3..=30u32 { + let prev = *chain.last().expect("non-empty"); + chain.push(block_with_root( + &prev, + TxMerkleNode::all_zeros(), + 1000 + height, + 0, + )); + } + + // As in `a_proof_for_another_block_is_not_a_verdict_on_ours`: our block at height 2 is already + // in the cache and the chain, so `GetHeader` does not catch the disagreement first, and the + // agreeing tip keeps any chain job from rewriting that height. + let mut cache = Cache::default(); + cache.headers.insert(ours.block_hash(), ours); + let cp = CheckPoint::new(BlockId { + height: 0, + hash: genesis.block_hash(), + }) + .insert(BlockId { + height: 2, + hash: ours.block_hash(), + }) + .insert(BlockId { + height: 30, + hash: chain[30].block_hash(), + }); + let mut state = new_state_with_cp(cache, descriptor, cp); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: chain, + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + let mut served = 0; + while let Some(req) = queue.pop_front() { + served += 1; + assert!(served < 200, "an error must not become a request loop"); + let resp = if req.method.as_ref() == "blockchain.transaction.get_merkle" { + raw_msg(json!({ + "jsonrpc": "2.0", + "id": req.id, + "error": { "code": 1, "message": "tx not found or is unconfirmed" }, + })) + } else { + response(&req, &server) + }; + state.poll(&mut queue, resp)?; + } + assert!( + state.cache().tx_cache.anchors.is_empty(), + "an error must not anchor anything" + ); + + // The server comes back to our block at that height. The tip is untouched, so this + // notification is the whole of the recovery. + server.headers[2] = ours; + let status = ElectrumScriptStatus::from_history(&server.history(&json!(spk_hash.to_string()))) + .expect("history must be non-empty"); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.scripthash.subscribe", + "params": [spk_hash.to_string(), status.to_string()], + })), + )?; + let updates = drain_requests(&mut state, &mut queue, &server); + assert!( + updates + .iter() + .any(|u| u.tx_update.anchors.contains(&(anchor_of(&ours, 2), txid))), + "the anchor must be reachable once the server proves it again" + ); + Ok(()) +} + +/// A job needs both the transactions and their anchors, and the server answers in any order. +/// +/// Every other test lets the `GetTx` land first, which puts the anchors on the job's final pass +/// with nothing running after them. This forces the other order: the merkle proof arrives first, +/// so the anchor resolves early and the job runs again when the transaction finally lands. +/// +/// Anchors are re-resolved from scratch on every pass, so that later pass must not lose the +/// anchor already in hand — the set of what to anchor is the question, not the answer, and +/// clearing it once answered would have the next pass ask an empty question and stage nothing. +#[test] +fn anchor_survives_a_pass_that_happens_after_it_resolved() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let server = Server { + headers: vec![genesis, header_1, header_2], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + let updates = drain_requests_proofs_first(&mut state, &mut queue, &server); + assert!( + updates.iter().any(|u| u + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid))), + "the anchor must still be emitted after a later pass" + ); + Ok(()) +} + +/// A tip notification landing while the previous one's headers are still in flight. +/// +/// The replacement job asks for the same heights, so its request is byte-identical to the one +/// already out. Deduplicated against that one, nothing new is sent, and the answer already on its +/// way describes the chain the server has just left. +#[test] +fn a_tip_that_moves_while_headers_are_in_flight_is_not_lost() -> anyhow::Result<()> { + let (descriptor, _spk_hash, _spk) = tracked_descriptor()?; + let (genesis, header_1) = base_headers(); + let build = |nonce: u32| { + let mut chain = vec![genesis, header_1]; + for height in 2..=3 { + let prev = *chain.last().expect("non-empty"); + chain.push(block_with_root( + &prev, + TxMerkleNode::all_zeros(), + 1000 + height, + nonce, + )); + } + chain + }; + let (chain_a, chain_b) = (build(0), build(1)); + assert_ne!(chain_a[3].block_hash(), chain_b[3].block_hash()); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: chain_a.clone(), + txs: Vec::new(), + merkle_proof: (Vec::new(), 0), + }; + + let notify = |chain: &[block::Header]| { + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(chain.last().expect("non-empty")), "height": 3 }], + })) + }; + + // The A-chain tip. Hold its headers request so the job is still waiting. + state.poll(&mut queue, notify(&chain_a))?; + let held = queue + .drain(..) + .filter(|req| req.method.as_ref() == "blockchain.block.headers") + .collect::>(); + assert!(!held.is_empty(), "the job must have asked for headers"); + let held_resp = held + .iter() + .map(|req| response(req, &server)) + .collect::>(); + + // The server reorgs to B and notifies the new tip at the same height. + server.headers = chain_b.clone(); + state.poll(&mut queue, notify(&chain_b))?; + assert!( + queue + .iter() + .any(|req| req.method.as_ref() == "blockchain.block.headers"), + "the replacement job must send its own request rather than adopt the one in flight" + ); + + // The held answer describes the chain the server has left. + let mut updates = Vec::new(); + for resp in held_resp { + updates.extend(state.poll(&mut queue, resp)?); + } + updates.extend(drain_requests(&mut state, &mut queue, &server)); + + let tip = updates + .iter() + .rev() + .find_map(|u| u.chain_update.clone()) + .expect("must get a chain update"); + assert_eq!( + tip.hash(), + chain_b[3].block_hash(), + "the local chain must end on the tip the server actually has" + ); + Ok(()) +} + +/// A headers batch answered from a chain other than the one announced. +/// +/// `blockchain.block.headers` is answered from whichever chain the server holds when it *replies*, +/// so a reorg between receiving the request and answering it returns blocks for a tip we were +/// never told about. Adopting them would put the checkpoint chain on a chain no notification ever +/// announced, and no notification would arrive to correct it. +#[test] +fn headers_for_a_chain_we_were_not_told_about_are_not_adopted() -> anyhow::Result<()> { + let (descriptor, _spk_hash, _spk) = tracked_descriptor()?; + let (genesis, h1) = base_headers(); + let h2 = block_with_root(&h1, TxMerkleNode::all_zeros(), 200, 0); + let a3 = block_with_root(&h2, TxMerkleNode::all_zeros(), 300, 0); + let b3 = block_with_root(&h2, TxMerkleNode::all_zeros(), 300, 1); + assert_ne!(a3.block_hash(), b3.block_hash()); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, h1, h2], + txs: Vec::new(), + merkle_proof: (Vec::new(), 0), + }; + state.init(&mut queue); + drain_requests(&mut state, &mut queue, &server); + + // A3 is announced, so that is the block the job is created to reach. + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&a3), "height": 3 }], + })), + )?; + + // But by the time the server answers, it is on B3 — and it does not announce it, because + // this response is the reorg's only appearance. + server.headers = vec![genesis, h1, h2, b3]; + let updates = drain_requests(&mut state, &mut queue, &server); + + let tip = updates.iter().rev().find_map(|u| u.chain_update.clone()); + assert_ne!( + tip.as_ref().map(|cp| cp.hash()), + Some(b3.block_hash()), + "a chain no notification announced must not be adopted" + ); + assert_ne!( + tip.as_ref().map(|cp| cp.hash()), + Some(a3.block_hash()), + "and the announced block was never actually delivered" + ); + Ok(()) +} + +/// A reorg has to re-verify the anchors of *every* script, not just whichever one notified last. +/// +/// Anchor scope is the set of script statuses [`ConfirmationJob`] was told to cover. Taking that from +/// the spk jobs which happen to be live makes it collapse: the jobs are cleared each time an +/// update is emitted, so a single notification arriving between updates narrows the scope to +/// that one script, and a reorg landing afterwards leaves every other script anchored to a block +/// which is no longer ours — with no notification coming to say so, since a transaction that +/// keeps its height keeps its script status. +#[test] +fn a_reorg_reanchors_every_script_not_just_the_last_to_notify() -> anyhow::Result<()> { + let (descriptor, spk_hash_a, spk_a) = tracked_descriptor()?; + let spk_b = descriptor.at_derivation_index(1)?.script_pubkey(); + + let (tx_a, tx_b) = (tx_paying(&spk_a, 50_000), tx_paying(&spk_b, 60_000)); + let (txid_a, txid_b) = (tx_a.compute_txid(), tx_b.compute_txid()); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid_a, 200, 0); + let header_3 = block_with_tx(&header_2, txid_b, 300, 0); + // The reorg keeps both txs at their heights — so neither script status changes — but moves + // them into different blocks, and extends the chain by one. + let header_2b = block_with_tx(&header_1, txid_a, 222, 1); + let header_3b = block_with_tx(&header_2b, txid_b, 333, 1); + let header_4b = block_with_root(&header_3b, TxMerkleNode::all_zeros(), 400, 0); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1, header_2, header_3], + txs: vec![(tx_a, 2), (tx_b, 3)], + merkle_proof: (Vec::new(), 0), + }; + + let anchors_of = |updates: &[Update<&'static str>]| { + updates + .iter() + .flat_map(|u| u.tx_update.anchors.iter().copied()) + .collect::>() + }; + + state.init(&mut queue); + let anchors = anchors_of(&drain_requests(&mut state, &mut queue, &server)); + assert!( + anchors.contains(&(anchor_of(&header_2, 2), txid_a)), + "tx_a must first be anchored" + ); + assert!( + anchors.contains(&(anchor_of(&header_3, 3), txid_b)), + "tx_b must first be anchored" + ); + + // Script A is re-notified with the status it already has — something a server does freely, + // and which says nothing about script B. + let status_a = + ElectrumScriptStatus::from_history(&server.history(&json!(spk_hash_a.to_string()))) + .expect("history must be non-empty"); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.scripthash.subscribe", + "params": [spk_hash_a.to_string(), status_a.to_string()], + })), + )?; + drain_requests(&mut state, &mut queue, &server); + + // The reorg lands. Only the tip announcement reports it; neither script will be notified. + server.headers = vec![genesis, header_1, header_2b, header_3b, header_4b]; + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_4b), "height": 4 }], + })), + )?; + let anchors = anchors_of(&drain_requests(&mut state, &mut queue, &server)); + + assert!( + anchors.contains(&(anchor_of(&header_2b, 2), txid_a)), + "the script that notified must be re-anchored against the new chain" + ); + assert!( + anchors.contains(&(anchor_of(&header_3b, 3), txid_b)), + "and so must every other script, which no notification will ever mention" + ); + Ok(()) +} + +/// A history that does not hash to the status the job is waiting for must not be re-asked for. +/// +/// The server has moved on since it notified — a reorg, or the transaction dropped out — so the +/// answer will be the same every time. Asking again is an unbounded request loop against the +/// server, and the notification carrying the status it actually holds is already on its way. +/// +/// The empty history is the sharpest version: `ElectrumScriptStatus::from_history` yields +/// nothing for it, so there is not even a status to compare against what was stored. +#[test] +fn a_history_that_cannot_match_the_job_is_not_re_asked() -> anyhow::Result<()> { + let (descriptor, spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + // The server's chain is one block long, so its history for this script is empty — it will + // never answer with the status the notification below carries. + let server = Server { + headers: vec![genesis, header_1], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + drain_requests(&mut state, &mut queue, &server); + + let status = + ElectrumScriptStatus::from_history(&[response::Tx::Confirmed(response::ConfirmedTx { + txid, + height: absolute::Height::from_consensus(2)?, + })]) + .expect("history is not empty"); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.scripthash.subscribe", + "params": [spk_hash.to_string(), status.to_string()], + })), + )?; + + let mut served = 0; + while let Some(req) = queue.pop_front() { + served += 1; + assert!( + served < 50, + "the client must not re-ask forever: {served} requests, last was {} {:?}", + req.method, + req.params + ); + state.poll(&mut queue, response(&req, &server))?; + } + Ok(()) +} + +/// The confirmation job must not wait on transactions it never reads. +/// +/// It works from the heights a history names, so once every script has its history it already +/// knows every block it needs. Holding it until the scripts finish downloading the transactions +/// in those histories serialises the header and proof fetches behind those downloads for +/// nothing — on a wallet with many scripts that is the whole sync sitting idle. +/// +/// The tip's own header arrives with the notification, so the one height here needs no header +/// request; reaching the proof is the proof that the job ran. +/// +/// Running ahead is not publishing ahead. The transactions a script is still downloading belong +/// in the same update as their anchors, so the finished job holds it until every script is done +/// — otherwise a caller sees an anchor for a transaction it was never given. +#[test] +fn confirmation_job_runs_ahead_but_the_update_waits_for_the_scripts() -> anyhow::Result<()> { + let (descriptor, spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + let txid = tx.compute_txid(); + let (genesis, header_1) = base_headers(); + let header_2 = block_with_tx(&header_1, txid, 200, 0); + + // Deliberately not seeded with the transaction, so the spk job has to ask for it. + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let mut server = Server { + headers: vec![genesis, header_1], + txs: vec![(tx, 2)], + merkle_proof: (Vec::new(), 0), + }; + + state.init(&mut queue); + drain_requests(&mut state, &mut queue, &server); + + let status = + ElectrumScriptStatus::from_history(&[response::Tx::Confirmed(response::ConfirmedTx { + txid, + height: absolute::Height::from_consensus(2)?, + })]) + .expect("history is not empty"); + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.scripthash.subscribe", + "params": [spk_hash.to_string(), status.to_string()], + })), + )?; + state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_2), "height": 2 }], + })), + )?; + server.headers.push(header_2); + + // Answer the history and nothing else, so the script's job is left in `ProcessingTxs`. + // Everything queued in response — including whatever the confirmation job asks for — is set + // aside unanswered. + let mut deferred = Vec::::new(); + while let Some(req) = queue.pop_front() { + if req.method.as_ref() == "blockchain.scripthash.get_history" { + state.poll(&mut queue, response(&req, &server))?; + } else { + deferred.push(req); + } + } + + assert!( + deferred + .iter() + .any(|req| req.method.as_ref() == "blockchain.transaction.get"), + "the script must still be waiting on the transaction its history named", + ); + assert!( + deferred + .iter() + .any(|req| req.method.as_ref() == "blockchain.transaction.get_merkle"), + "the confirmation job must reach proof fetching without waiting for that transaction", + ); + + // Let the confirmation job finish: answer everything, still except the transaction. + let mut updates = Vec::new(); + let mut tx_reqs = Vec::::new(); + let mut pending = deferred; + while let Some(req) = pending.pop() { + if req.method.as_ref() == "blockchain.transaction.get" { + tx_reqs.push(req); + continue; + } + if let Some(update) = state.poll(&mut queue, response(&req, &server))? { + updates.push(update); + } + pending.extend(queue.drain(..)); + } + assert!( + updates.is_empty(), + "nothing may be published while a script is still downloading its transactions", + ); + + // The transaction finally arrives, and with it the whole update. + for req in tx_reqs { + if let Some(update) = state.poll(&mut queue, response(&req, &server))? { + updates.push(update); + } + } + updates.extend(drain_requests(&mut state, &mut queue, &server)); + + let update = match updates.as_slice() { + [update] => update, + other => panic!("exactly one update must be published, got {}", other.len()), + }; + assert!( + update + .tx_update + .txs + .iter() + .any(|t| t.compute_txid() == txid), + "the update must carry the transaction", + ); + assert!( + update + .tx_update + .anchors + .contains(&(anchor_of(&header_2, 2), txid)), + "the update must carry its anchor alongside it", + ); + + // A finished job must hand its update over once, not on every poll that reaches it. The + // server re-announcing the tip it already announced drives `poll_confirmation_job` without + // moving the target or the statuses, so nothing may come back out. + let again = state.poll( + &mut queue, + raw_msg(json!({ + "jsonrpc": "2.0", + "method": "blockchain.headers.subscribe", + "params": [{ "hex": serialize_hex(&header_2), "height": 2 }], + })), + )?; + assert!(again.is_none(), "an update must not be handed over twice"); + assert!( + drain_requests(&mut state, &mut queue, &server).is_empty(), + "a job with nothing left to do must not republish", + ); + Ok(()) +} + +/// A transaction is cached under the txid it was asked for, never under the one it claims. +/// +/// Nothing downstream can catch a substitution: the prevouts of the wrong transaction resolve +/// into `txouts` as if they were the right one's, and a caller sees inputs that were never +/// spent. `SpkJob::poll` errors only in the narrow case where the substitute is too short to +/// reach a spent vout, which a server picking any longer transaction sails past. +#[test] +fn a_transaction_that_is_not_the_one_asked_for_is_rejected() -> anyhow::Result<()> { + let (descriptor, _spk_hash, spk) = tracked_descriptor()?; + let tx = tx_paying(&spk, 50_000); + // Same shape, different value, so it is a perfectly valid transaction with another txid. + let impostor = tx_paying(&spk, 60_000); + assert_ne!(impostor.compute_txid(), tx.compute_txid()); + let (genesis, header_1) = base_headers(); + + let mut state = new_state(Cache::default(), descriptor, genesis); + let mut queue = ReqQueue::new(); + let server = Server { + headers: vec![genesis, header_1], + txs: vec![(tx, 1)], + merkle_proof: (Vec::new(), 0), + }; + + // Answer the initial sync honestly, except that every transaction comes back as the + // impostor. The subscribe response carries the status, so this drives the whole flow. + state.init(&mut queue); + let mut substituted = false; + let mut result = Ok(None); + while let Some(req) = queue.pop_front() { + let msg = if req.method.as_ref() == "blockchain.transaction.get" { + substituted = true; + raw_msg(json!({ + "jsonrpc": "2.0", + "id": req.id, + "result": serialize_hex(&impostor), + })) + } else { + response(&req, &server) + }; + result = state.poll(&mut queue, msg); + if result.is_err() { + break; + } + } + assert!(substituted, "the test must have answered a `GetTx`"); + assert!( + result.is_err(), + "a transaction that is not the one asked for must not be accepted", + ); + Ok(()) +}