diff --git a/app/app.rs b/app/app.rs index 5cfc2446..f9247c0f 100644 --- a/app/app.rs +++ b/app/app.rs @@ -76,8 +76,12 @@ fn update_wallet(node: &Node, wallet: &Wallet) -> Result<(), Error> { .into_iter() .map(|(outpoint, spent_output)| (outpoint, spent_output.inpoint)) .collect(); - wallet.put_utxos(&utxos)?; - wallet.spend_utxos(&spent)?; + wallet.sync_confirmed(&utxos, &spent)?; + let confirmed: HashSet = + wallet.get_utxos()?.into_keys().collect(); + let (unconfirmed, mempool_spent) = + node.get_mempool_view(&addresses, &confirmed)?; + wallet.set_mempool_view(&unconfirmed, &mempool_spent)?; tracing::debug!("finished wallet update"); Ok(()) @@ -88,14 +92,34 @@ fn update( node: &Node, utxos: &mut HashMap, wallet: &Wallet, + spend_zero_conf_change: bool, ) -> Result<(), Error> { tracing::trace!("Updating wallet"); let () = update_wallet(node, wallet)?; - *utxos = wallet.get_utxos()?; + *utxos = spendable_utxos(wallet, spend_zero_conf_change)?; tracing::trace!("Updated wallet"); Ok(()) } +/// The coins the wallet may put in a new transaction. It matches what +/// `Wallet::select_coins` takes under the same option. +fn spendable_utxos( + wallet: &Wallet, + spend_zero_conf_change: bool, +) -> Result, Error> { + let mut utxos = wallet.get_utxos()?; + let spent = wallet.get_mempool_spent_utxos()?; + if spend_zero_conf_change { + utxos.extend(wallet.get_unconfirmed_utxos()?); + } + // A withdrawal output belongs to a bundle, and the state refuses a + // transaction that spends one, so `select_coins` skips it too. + utxos.retain(|outpoint, output| { + !spent.contains(outpoint) && !output.content.is_withdrawal() + }); + Ok(utxos) +} + struct ProtoSupport { block_producer: bool, miner: bool, @@ -122,6 +146,7 @@ pub struct Config { pub network: thunder::types::Network, pub network_magic_override: Option, pub server_names: HashSet, + pub spend_zero_conf_change: bool, pub wallet_dir: PathBuf, } @@ -131,6 +156,7 @@ pub struct App { pub wallet: Wallet, pub miner: Option>>, pub utxos: Arc>>, + pub spend_zero_conf_change: bool, task: Arc>, pub transaction: Arc>, pub runtime: Arc, @@ -142,10 +168,20 @@ impl App { node: Arc, utxos: Arc>>, wallet: Wallet, + spend_zero_conf_change: bool, ) -> Result<(), Error> { - let mut state_changes = node.watch_state(); - while let Some(()) = state_changes.next().await { - let () = update(&node, &mut utxos.write(), &wallet)?; + // The mempool signal matters as much as the tip: a payment that a peer + // sends appears without a block, and a coin that a transaction spends + // leaves the spendable set the moment the mempool takes it. + let mut changes = + futures::stream::select(node.watch_state(), node.watch_mempool()); + while let Some(()) = changes.next().await { + let () = update( + &node, + &mut utxos.write(), + &wallet, + spend_zero_conf_change, + )?; } Ok(()) } @@ -154,11 +190,15 @@ impl App { node: Arc, utxos: Arc>>, wallet: Wallet, + spend_zero_conf_change: bool, ) -> JoinHandle<()> { - spawn(Self::task(node, utxos, wallet).unwrap_or_else(|err| { - let err = anyhow::Error::from(err); - tracing::error!("{err:#}") - })) + spawn( + Self::task(node, utxos, wallet, spend_zero_conf_change) + .unwrap_or_else(|err| { + let err = anyhow::Error::from(err); + tracing::error!("{err:#}") + }), + ) } async fn check_status_serving( @@ -336,26 +376,27 @@ impl App { &mut rng, &runtime, )?; - let utxos = { - let mut utxos = wallet.get_utxos()?; - let transactions = node.get_all_transactions()?; - for transaction in &transactions { - for (outpoint, _) in &transaction.transaction.inputs { - utxos.remove(outpoint); - } - } - Arc::new(RwLock::new(utxos)) - }; let node = Arc::new(node); + let spend_zero_conf_change = config.spend_zero_conf_change; + let () = update_wallet(&node, &wallet)?; + let utxos = Arc::new(RwLock::new(spendable_utxos( + &wallet, + spend_zero_conf_change, + )?)); let miner = miner.map(|miner| Arc::new(TokioRwLock::new(miner))); - let task = - Self::spawn_task(node.clone(), utxos.clone(), wallet.clone()); + let task = Self::spawn_task( + node.clone(), + utxos.clone(), + wallet.clone(), + spend_zero_conf_change, + ); drop(rt_guard); Ok(Self { node, wallet, miner, utxos, + spend_zero_conf_change, task: Arc::new(task), transaction: Arc::new(RwLock::new(Transaction { inputs: vec![].into(), @@ -369,7 +410,12 @@ impl App { /// Update utxos & wallet fn update(&self) -> Result<(), Error> { - update(self.node.as_ref(), &mut self.utxos.write(), &self.wallet) + update( + self.node.as_ref(), + &mut self.utxos.write(), + &self.wallet, + self.spend_zero_conf_change, + ) } /// Regenerate proofs and submit transaction diff --git a/app/cli.rs b/app/cli.rs index ceabbfb1..8a208583 100644 --- a/app/cli.rs +++ b/app/cli.rs @@ -28,6 +28,7 @@ pub struct Config { pub private_rpc_addr: SocketAddr, pub rpc_addr: SocketAddr, pub server_names: HashSet, + pub spend_zero_conf_change: bool, pub wallet_dir: PathBuf, } @@ -50,6 +51,7 @@ impl Config { private_rpc_addr, rpc_addr, server_names, + spend_zero_conf_change, wallet_dir, } = self; let add_peers = std::fmt::from_fn(|f| { @@ -79,6 +81,7 @@ impl Config { %private_rpc_addr, %rpc_addr, ?server_names, + %spend_zero_conf_change, wallet_dir = %wallet_dir.display(), msg, ) @@ -232,6 +235,12 @@ pub(super) struct Cli { /// This option can be specified multiple times. #[arg(long = "server-name")] server_names: Vec, + /// Spend the wallet's own unconfirmed change. The wallet takes an output + /// only when it funded every input of the transaction that made it, so an + /// unconfirmed payment from someone else waits for a block. This is the + /// rule that Bitcoin Core calls `-spendzeroconfchange`. + #[arg(default_value_t = true, long, action = clap::ArgAction::Set)] + spend_zero_conf_change: bool, /// Data directory for storing wallet data #[arg(long)] wallet_dir: Option, @@ -276,6 +285,7 @@ impl Cli { private_rpc_addr: self.private_rpc_addr, rpc_addr: self.rpc_addr, server_names: HashSet::from_iter(self.server_names), + spend_zero_conf_change: self.spend_zero_conf_change, wallet_dir, }) } diff --git a/app/gui/coins/transfer_receive.rs b/app/gui/coins/transfer_receive.rs index d2a6a036..a5493137 100644 --- a/app/gui/coins/transfer_receive.rs +++ b/app/gui/coins/transfer_receive.rs @@ -17,9 +17,13 @@ fn create_transfer( fee: bitcoin::Amount, ) -> anyhow::Result<()> { let accumulator = app.node.get_tip_accumulator()?; - let tx = app - .wallet - .create_transaction(&accumulator, dest, amount, fee)?; + let tx = app.wallet.create_transaction( + &accumulator, + app.spend_zero_conf_change, + dest, + amount, + fee, + )?; app.sign_and_send(tx)?; Ok(()) } diff --git a/app/gui/coins/tx_builder.rs b/app/gui/coins/tx_builder.rs index 6ef4975c..069c2b0c 100644 --- a/app/gui/coins/tx_builder.rs +++ b/app/gui/coins/tx_builder.rs @@ -53,6 +53,13 @@ impl TxBuilder { ui.monospace("outpoint"); ui.monospace("value"); ui.end_row(); + // A mempool transaction may spend a chosen coin between two + // frames, which drops it from the spendable set. Drop it here, so + // the grid, the value and the transaction all agree. + self.base_tx + .inputs + .0 + .retain(|(outpoint, _)| utxos_read.contains_key(outpoint)); let mut remove = None; for (vout, (outpoint, _)) in self.base_tx.inputs.iter().enumerate() { diff --git a/app/gui/mod.rs b/app/gui/mod.rs index ef1f746d..93ebc62e 100644 --- a/app/gui/mod.rs +++ b/app/gui/mod.rs @@ -2,7 +2,7 @@ use std::task::Poll; use eframe::egui::{self, RichText}; use strum::{EnumIter, IntoEnumIterator}; -use thunder::{util::Watchable, wallet::Wallet}; +use thunder::{types::wallet::Balance, util::Watchable, wallet::Wallet}; use util::{BITCOIN_LOGO_FA, BITCOIN_ORANGE, show_btc_amount}; use crate::{app::App, line_buffer::LineBuffer, util::PromiseStream}; @@ -86,7 +86,7 @@ struct BottomPanel { initialized: Option, /// None if uninitialized /// Some(None) if failed to initialize - balance: Option>, + balance: Option>, } impl BottomPanel { @@ -107,8 +107,12 @@ impl BottomPanel { let rt_guard = initialized.app.runtime.enter(); match initialized.wallet_updated.poll_next() { Some(Poll::Ready(())) => { - self.balance = match initialized.app.wallet.get_balance() { - Ok(balance) => Some(Some(balance.total)), + self.balance = match initialized + .app + .wallet + .get_balance(initialized.app.spend_zero_conf_change) + { + Ok(balance) => Some(Some(balance)), Err(err) => { let err = anyhow::Error::from(err); tracing::error!("Failed to update balance: {err:#}"); @@ -122,7 +126,7 @@ impl BottomPanel { } fn show_balance(&self, ui: &mut egui::Ui) { - match self.balance { + match &self.balance { Some(Some(balance)) => { ui.monospace( RichText::new(BITCOIN_LOGO_FA.to_string()) @@ -130,8 +134,26 @@ impl BottomPanel { ); ui.monospace_selectable_singleline( false, - format!("Balance: {}", show_btc_amount(balance)), + format!("Balance: {}", show_btc_amount(balance.total)), ); + if balance.unconfirmed != bitcoin::Amount::ZERO { + ui.monospace_selectable_singleline( + false, + format!( + "Unconfirmed: {}", + show_btc_amount(balance.unconfirmed) + ), + ); + } + if balance.available != balance.total { + ui.monospace_selectable_singleline( + false, + format!( + "Available: {}", + show_btc_amount(balance.available) + ), + ); + } } Some(None) => { ui.monospace_selectable_singleline( diff --git a/app/gui/parent_chain/transfer.rs b/app/gui/parent_chain/transfer.rs index 41247dee..23b9e948 100644 --- a/app/gui/parent_chain/transfer.rs +++ b/app/gui/parent_chain/transfer.rs @@ -143,6 +143,7 @@ fn create_withdrawal( let accumulator = app.node.get_tip_accumulator()?; let tx = app.wallet.create_withdrawal( &accumulator, + app.spend_zero_conf_change, mainchain_address, amount, mainchain_fee, diff --git a/app/main.rs b/app/main.rs index 21ea47d5..8b71f3b8 100644 --- a/app/main.rs +++ b/app/main.rs @@ -228,6 +228,7 @@ fn main() -> anyhow::Result<()> { network: config.network, network_magic_override: config.network_magic_override, server_names: config.server_names, + spend_zero_conf_change: config.spend_zero_conf_change, wallet_dir: config.wallet_dir, }; let app = app::App::new(app_config).inspect(|app| { diff --git a/app/rpc_server.rs b/app/rpc_server.rs index 82b0392f..5b7d0941 100644 --- a/app/rpc_server.rs +++ b/app/rpc_server.rs @@ -376,7 +376,10 @@ impl rpc_api::node::RpcServer #[async_trait] impl rpc_api::wallet::RpcServer for RpcServerImpl { async fn balance(&self) -> RpcResult { - self.app.wallet.get_balance().map_err(custom_err) + self.app + .wallet + .get_balance(self.app.spend_zero_conf_change) + .map_err(custom_err) } async fn create_deposit( @@ -411,6 +414,7 @@ impl rpc_api::wallet::RpcServer for RpcServerImpl { .wallet .create_transaction( &accumulator, + self.app.spend_zero_conf_change, dest, Amount::from_sat(value_sats), Amount::from_sat(fee_sats), @@ -435,6 +439,7 @@ impl rpc_api::wallet::RpcServer for RpcServerImpl { .wallet .create_withdrawal( &accumulator, + self.app.spend_zero_conf_change, mainchain_address, Amount::from_sat(amount_sats), Amount::from_sat(mainchain_fee_sats), @@ -506,6 +511,21 @@ impl rpc_api::wallet::RpcServer for RpcServerImpl { Ok(utxos) } + async fn get_unconfirmed_wallet_utxos( + &self, + ) -> RpcResult> { + let utxos = self + .app + .wallet + .get_unconfirmed_utxos() + .map_err(custom_err)?; + let utxos = utxos + .into_iter() + .map(|(outpoint, output)| PointedOutput { outpoint, output }) + .collect(); + Ok(utxos) + } + async fn mine(&self, fee: Option) -> RpcResult<()> { let fee = fee.map(bitcoin::Amount::from_sat); self.app diff --git a/cli/lib.rs b/cli/lib.rs index 085eb600..ae7cce47 100644 --- a/cli/lib.rs +++ b/cli/lib.rs @@ -111,6 +111,8 @@ pub enum Command { GetWalletAddresses, /// Get wallet UTXOs GetWalletUtxos, + /// Get the unconfirmed wallet UTXOs that the wallet may spend + GetUnconfirmedWalletUtxos, /// Get withdrawal bundle by M6id GetWithdrawalBundle { m6id: M6id }, /// Invalidate a block, potentially re-orging to a valid ancestor of the @@ -311,6 +313,10 @@ where let utxos = rpc_client.get_wallet_utxos().await?; serde_json::to_string_pretty(&utxos)? } + Command::GetUnconfirmedWalletUtxos => { + let utxos = rpc_client.get_unconfirmed_wallet_utxos().await?; + serde_json::to_string_pretty(&utxos)? + } Command::GetWithdrawalBundle { m6id } => { let withdrawal_bundle = rpc_client.get_withdrawal_bundle(m6id).await?; diff --git a/integration_tests/block_template.rs b/integration_tests/block_template.rs index 1b7d21bf..1de7846b 100644 --- a/integration_tests/block_template.rs +++ b/integration_tests/block_template.rs @@ -55,6 +55,7 @@ async fn block_template_task( Init { thunder_app: bin_paths.thunder()?.clone(), data_dir_suffix: None, + extra_args: Vec::new(), }, &enforcer_post_setup, res_tx, diff --git a/integration_tests/ibd.rs b/integration_tests/ibd.rs index 1bb2b89e..60796387 100644 --- a/integration_tests/ibd.rs +++ b/integration_tests/ibd.rs @@ -48,6 +48,7 @@ async fn setup( Init { thunder_app: bin_paths.thunder()?.clone(), data_dir_suffix: Some("sender".to_owned()), + extra_args: Vec::new(), }, &enforcer_post_setup, res_tx.clone(), @@ -58,6 +59,7 @@ async fn setup( Init { thunder_app: bin_paths.thunder()?.clone(), data_dir_suffix: Some("syncer".to_owned()), + extra_args: Vec::new(), }, &enforcer_post_setup, res_tx, diff --git a/integration_tests/integration_test.rs b/integration_tests/integration_test.rs index 5a8e5b9d..a605e185 100644 --- a/integration_tests/integration_test.rs +++ b/integration_tests/integration_test.rs @@ -15,6 +15,7 @@ use crate::{ block_template::block_template_trial, ibd::{ibd_trial, reorg_across_deposit_trial}, setup::{Init, PostSetup}, + spend_unconfirmed::{spend_unconfirmed_off_trial, spend_unconfirmed_trial}, unknown_withdrawal::unknown_withdrawal_trial, util::BinPaths, }; @@ -146,6 +147,7 @@ fn deposit_withdraw_roundtrip_trial( Init { thunder_app: bin_paths.thunder()?.clone(), data_dir_suffix: None, + extra_args: Vec::new(), }, res_tx, ) @@ -183,6 +185,16 @@ pub fn tests( file_registry.clone(), failure_collector.clone(), ), + spend_unconfirmed_trial( + bin_paths.clone(), + file_registry.clone(), + failure_collector.clone(), + ), + spend_unconfirmed_off_trial( + bin_paths.clone(), + file_registry.clone(), + failure_collector.clone(), + ), unknown_withdrawal_trial(bin_paths, file_registry, failure_collector), ] } diff --git a/integration_tests/main.rs b/integration_tests/main.rs index a8b80495..f2147f8d 100644 --- a/integration_tests/main.rs +++ b/integration_tests/main.rs @@ -8,6 +8,7 @@ mod block_template; mod ibd; mod integration_test; mod setup; +mod spend_unconfirmed; mod unknown_withdrawal; mod util; diff --git a/integration_tests/setup.rs b/integration_tests/setup.rs index da03e01f..dae177d4 100644 --- a/integration_tests/setup.rs +++ b/integration_tests/setup.rs @@ -38,6 +38,9 @@ impl ReservedPorts { pub struct Init { pub thunder_app: PathBuf, pub data_dir_suffix: Option, + /// More arguments for the thunder binary, such as + /// `--spend-zero-conf-change=false`. + pub extra_args: Vec, } #[derive(Debug, Error)] @@ -178,12 +181,17 @@ impl Sidechain for PostSetup { rpc_port: reserved_ports.rpc.port(), }; let thunder_app_task = thunder_app - .spawn_command_with_args::([], [], { - let res_tx = res_tx.clone(); - move |err| { - let _err: Result<(), _> = res_tx.unbounded_send(Err(err)); - } - }); + .spawn_command_with_args::( + [], + init.extra_args, + { + let res_tx = res_tx.clone(); + move |err| { + let _err: Result<(), _> = + res_tx.unbounded_send(Err(err)); + } + }, + ); tracing::debug!("Started thunder"); sleep(Duration::from_secs(1)).await; let rpc_client = jsonrpsee::http_client::HttpClient::builder() diff --git a/integration_tests/spend_unconfirmed.rs b/integration_tests/spend_unconfirmed.rs new file mode 100644 index 00000000..3e688dfa --- /dev/null +++ b/integration_tests/spend_unconfirmed.rs @@ -0,0 +1,302 @@ +//! Test that the wallet spends an unconfirmed output, and that one block +//! carries the whole chain. + +use bip300301_enforcer_integration_tests::{ + integration_test::{ + activate_sidechain, deposit, fund_enforcer, propose_sidechain, + }, + setup::{ + Mode, Network, PostSetup as EnforcerPostSetup, + PreSetup as EnforcerPreSetup, SetupOpts as EnforcerSetupOpts, + Sidechain as _, + }, + util::{ + AbortOnDrop, AsyncTrial, BinPaths as EnforcerBinPaths, + TestFailureCollector, TestFileRegistry, + }, +}; +use std::future::Future; + +use bitcoin::Amount; +use futures::{ + FutureExt as _, StreamExt as _, channel::mpsc, future::BoxFuture, +}; +use thunder_app_rpc_api::{node::RpcClient as _, wallet::RpcClient as _}; +use tracing::Instrument as _; + +use crate::{ + setup::{Init, PostSetup}, + util::BinPaths, +}; + +const DEPOSIT_AMOUNT: Amount = Amount::from_sat(21_000_000); +const DEPOSIT_FEE: Amount = Amount::from_sat(1_000_000); +const TRANSFER_FEE: Amount = Amount::from_sat(1_000); + +/// Initial setup for the test +async fn setup( + enforcer_bin_paths: &EnforcerBinPaths, + res_tx: mpsc::UnboundedSender>, +) -> anyhow::Result { + let enforcer_pre_setup = + EnforcerPreSetup::new(enforcer_bin_paths, Network::Regtest)?; + let mut enforcer_post_setup = { + let setup_opts: EnforcerSetupOpts = Default::default(); + enforcer_pre_setup + .setup(Mode::Mempool, setup_opts, res_tx.clone()) + .await? + }; + let () = propose_sidechain::(&mut enforcer_post_setup).await?; + let () = activate_sidechain::(&mut enforcer_post_setup).await?; + let () = fund_enforcer::(&mut enforcer_post_setup).await?; + Ok(enforcer_post_setup) +} + +async fn spend_unconfirmed_task( + bin_paths: BinPaths, + res_tx: mpsc::UnboundedSender>, +) -> anyhow::Result<()> { + let mut enforcer_post_setup = + setup(&bin_paths.others, res_tx.clone()).await?; + let mut sidechain = PostSetup::setup( + Init { + thunder_app: bin_paths.thunder()?.clone(), + data_dir_suffix: None, + extra_args: Vec::new(), + }, + &enforcer_post_setup, + res_tx, + ) + .await?; + + let deposit_address = sidechain.get_deposit_address().await?; + let () = deposit( + &mut enforcer_post_setup, + &mut sidechain, + &deposit_address, + DEPOSIT_AMOUNT, + DEPOSIT_FEE, + ) + .await?; + tracing::info!("Deposited to the sidechain"); + + let confirmed = sidechain.rpc_client.balance().await?; + anyhow::ensure!(confirmed.total == DEPOSIT_AMOUNT); + anyhow::ensure!(confirmed.unconfirmed == Amount::ZERO); + + // The deposit is one coin, so the first transfer takes all of it and the + // second one must spend the first one's change. + let dest = sidechain.rpc_client.get_new_address().await?; + let half = DEPOSIT_AMOUNT.to_sat() / 2; + let block_count = sidechain.rpc_client.getblockcount().await?; + + tracing::debug!("Sending the parent transfer"); + let parent = sidechain + .rpc_client + .create_transfer(dest, half, TRANSFER_FEE.to_sat()) + .await?; + anyhow::ensure!( + sidechain.rpc_client.getblockcount().await? == block_count, + "the parent must stay in the mempool", + ); + + let pending = sidechain.rpc_client.balance().await?; + anyhow::ensure!( + pending.unconfirmed > Amount::ZERO, + "the change of the parent must read as unconfirmed", + ); + anyhow::ensure!( + !sidechain + .rpc_client + .get_unconfirmed_wallet_utxos() + .await? + .is_empty(), + "the wallet must hold an unconfirmed output", + ); + + tracing::debug!("Sending the child transfer, which spends the change"); + let child = sidechain + .rpc_client + .create_transfer(dest, half / 2, TRANSFER_FEE.to_sat()) + .await?; + anyhow::ensure!(child != parent); + + for txid in [parent, child] { + let found = sidechain.rpc_client.get_transaction(txid).await?; + anyhow::ensure!( + found.is_some_and(|found| found.block_hash.is_none()), + "{txid} must sit in the mempool", + ); + } + + tracing::debug!("Checking that one template carries the whole chain"); + let template = sidechain.rpc_client.get_block_template().await?; + let body: Vec<_> = template + .block + .body + .transactions + .iter() + .map(|tx| tx.txid()) + .collect(); + anyhow::ensure!( + body == vec![parent, child], + "the parent must come first, the body holds {body:?}", + ); + + tracing::debug!("BMM one block, which must carry the parent and the child"); + let () = sidechain.bmm_single(&mut enforcer_post_setup).await?; + anyhow::ensure!( + sidechain.rpc_client.getblockcount().await? == block_count + 1 + ); + for txid in [parent, child] { + let found = sidechain.rpc_client.get_transaction(txid).await?; + anyhow::ensure!( + found.is_some_and(|found| found.block_hash.is_some()), + "the block must carry {txid}", + ); + } + let settled = sidechain.rpc_client.balance().await?; + anyhow::ensure!( + settled.unconfirmed == Amount::ZERO, + "nothing stays unconfirmed after the block", + ); + // The wallet pays itself, and the coinbase of the block it mines pays the + // fees back to it, so the whole deposit stays. + anyhow::ensure!( + settled.total == DEPOSIT_AMOUNT, + "the wallet paid itself and took the fees back: {settled:?}", + ); + + drop(sidechain); + drop(enforcer_post_setup.tasks); + tracing::info!( + "Removing {}", + enforcer_post_setup.directories.base_dir.path().display() + ); + enforcer_post_setup.directories.base_dir.cleanup()?; + Ok(()) +} + +/// With `--spend-zero-conf-change false` the wallet must still show the value +/// of its own unconfirmed change, and must refuse to spend it. Bitcoin Core +/// reports such value and refuses it under the same option. +async fn spend_unconfirmed_off_task( + bin_paths: BinPaths, + res_tx: mpsc::UnboundedSender>, +) -> anyhow::Result<()> { + let mut enforcer_post_setup = + setup(&bin_paths.others, res_tx.clone()).await?; + let mut sidechain = PostSetup::setup( + Init { + thunder_app: bin_paths.thunder()?.clone(), + data_dir_suffix: None, + extra_args: vec!["--spend-zero-conf-change=false".to_owned()], + }, + &enforcer_post_setup, + res_tx, + ) + .await?; + + let deposit_address = sidechain.get_deposit_address().await?; + let () = deposit( + &mut enforcer_post_setup, + &mut sidechain, + &deposit_address, + DEPOSIT_AMOUNT, + DEPOSIT_FEE, + ) + .await?; + + let confirmed = sidechain.rpc_client.balance().await?; + anyhow::ensure!(confirmed.total == DEPOSIT_AMOUNT); + anyhow::ensure!(confirmed.unconfirmed == Amount::ZERO); + + let dest = sidechain.rpc_client.get_new_address().await?; + let half = DEPOSIT_AMOUNT.to_sat() / 2; + let _parent = sidechain + .rpc_client + .create_transfer(dest, half, TRANSFER_FEE.to_sat()) + .await?; + + let pending = sidechain.rpc_client.balance().await?; + anyhow::ensure!( + pending.unconfirmed > Amount::ZERO, + "the change must stay visible, got {pending:?}", + ); + anyhow::ensure!( + pending.total == DEPOSIT_AMOUNT - TRANSFER_FEE, + "the wallet still holds the deposit less the fee, got {pending:?}", + ); + anyhow::ensure!( + pending.available == Amount::ZERO, + "the wallet may take nothing, got {pending:?}", + ); + let refused = sidechain + .rpc_client + .create_transfer(dest, half / 2, TRANSFER_FEE.to_sat()) + .await; + let Err(err) = refused else { + anyhow::bail!("the wallet must refuse to spend its unconfirmed change"); + }; + anyhow::ensure!( + err.to_string().contains("not enough funds"), + "the wallet refused for the wrong reason: {err}", + ); + + drop(sidechain); + drop(enforcer_post_setup.tasks); + enforcer_post_setup.directories.base_dir.cleanup()?; + Ok(()) +} + +/// Run one task under its own result channel, so an early failure inside the +/// task reaches the trial rather than the spawned task. +async fn run_task( + bin_paths: BinPaths, + task: impl FnOnce(BinPaths, mpsc::UnboundedSender>) -> Fut + + Send + + 'static, +) -> anyhow::Result<()> +where + Fut: Future> + Send, +{ + let (res_tx, mut res_rx) = mpsc::unbounded(); + let _test_task: AbortOnDrop<()> = tokio::task::spawn({ + let res_tx = res_tx.clone(); + async move { + let res = task(bin_paths, res_tx.clone()).await; + let _send_err: Result<(), _> = res_tx.unbounded_send(res); + } + .in_current_span() + }) + .into(); + res_rx.next().await.ok_or_else(|| { + anyhow::anyhow!("Unexpected end of test task result stream") + })? +} + +pub fn spend_unconfirmed_trial( + bin_paths: BinPaths, + file_registry: TestFileRegistry, + failure_collector: TestFailureCollector, +) -> AsyncTrial>> { + AsyncTrial::new( + "spend_unconfirmed", + run_task(bin_paths, spend_unconfirmed_task).boxed(), + file_registry, + failure_collector, + ) +} + +pub fn spend_unconfirmed_off_trial( + bin_paths: BinPaths, + file_registry: TestFileRegistry, + failure_collector: TestFailureCollector, +) -> AsyncTrial>> { + AsyncTrial::new( + "spend_unconfirmed_off", + run_task(bin_paths, spend_unconfirmed_off_task).boxed(), + file_registry, + failure_collector, + ) +} diff --git a/integration_tests/unknown_withdrawal.rs b/integration_tests/unknown_withdrawal.rs index 613cce30..df82187d 100644 --- a/integration_tests/unknown_withdrawal.rs +++ b/integration_tests/unknown_withdrawal.rs @@ -64,6 +64,7 @@ async fn unknown_withdrawal_task( Init { thunder_app: bin_paths.thunder()?.clone(), data_dir_suffix: Some("withdrawer".to_owned()), + extra_args: Vec::new(), }, &enforcer_post_setup, res_tx.clone(), @@ -95,6 +96,7 @@ async fn unknown_withdrawal_task( Init { thunder_app: bin_paths.thunder()?.clone(), data_dir_suffix: Some("successor".to_owned()), + extra_args: Vec::new(), }, &enforcer_post_setup, res_tx, diff --git a/lib/mempool.rs b/lib/mempool.rs index a1f779bb..64b70bca 100644 --- a/lib/mempool.rs +++ b/lib/mempool.rs @@ -1,4 +1,7 @@ -use std::{collections::VecDeque, path::PathBuf}; +use std::{ + collections::{HashMap, HashSet, VecDeque}, + path::PathBuf, +}; use fallible_iterator::FallibleIterator as _; use heed::types::SerdeBincode; @@ -6,11 +9,18 @@ use sneed::{ DatabaseUnique, DbError, EnvError, RoTxn, RwTxn, RwTxnError, UnitKey, db, }; -use crate::types::{ - Accumulator, AuthorizedTransaction, OutPoint, Txid, UtreexoError, VERSION, - Version, +use crate::{ + types::{ + Accumulator, Address, AuthorizedTransaction, OutPoint, Output, + Transaction, Txid, UtreexoError, VERSION, Version, + }, + util::Watchable, }; +/// Longest chain of unconfirmed transactions the mempool accepts. Bitcoin +/// Core holds the same number in `DEFAULT_ANCESTOR_LIMIT`. +pub const MAX_UNCONFIRMED_ANCESTORS: usize = 25; + #[allow(clippy::duplicated_attributes)] #[derive(Debug, thiserror::Error, transitive::Transitive)] #[transitive(from(db::error::TryGet, DbError))] @@ -31,6 +41,11 @@ pub enum Error { Utreexo(#[from] UtreexoError), #[error("can't add transaction, utxo double spent")] UtxoDoubleSpent, + #[error( + "can't add transaction, it has {count} unconfirmed ancestors and the \ + limit is {MAX_UNCONFIRMED_ANCESTORS}" + )] + TooManyAncestors { count: usize }, } #[derive(Clone)] @@ -86,6 +101,30 @@ impl MemPool { &self, txn: &mut RwTxn, transaction: &AuthorizedTransaction, + ) -> Result<(), Error> { + let ancestors = self.ancestors(txn, &transaction.transaction)?; + if ancestors.len() >= MAX_UNCONFIRMED_ANCESTORS { + return Err(Error::TooManyAncestors { + count: ancestors.len(), + }); + } + self.insert(txn, transaction) + } + + /// Take back a transaction that a disconnected block carried. The chain + /// accepted it once, so the ancestor limit does not apply. + pub fn put_disconnected( + &self, + txn: &mut RwTxn, + transaction: &AuthorizedTransaction, + ) -> Result<(), Error> { + self.insert(txn, transaction) + } + + fn insert( + &self, + txn: &mut RwTxn, + transaction: &AuthorizedTransaction, ) -> Result<(), Error> { let txid = transaction.transaction.txid(); tracing::debug!("adding transaction {txid} to mempool"); @@ -142,6 +181,231 @@ impl MemPool { Ok(()) } + /// Remove a transaction that a block confirms, and keep its children. A + /// child of a confirmed parent spends a confirmed output, so it stays + /// valid. + pub fn delete_confirmed( + &self, + rwtxn: &mut RwTxn, + txid: Txid, + ) -> Result<(), Error> { + let Some(tx) = self + .transactions + .try_get(rwtxn, &txid) + .map_err(DbError::from)? + else { + return Ok(()); + }; + for (outpoint, _) in &tx.transaction.inputs { + self.spent_utxos + .delete(rwtxn, outpoint) + .map_err(DbError::from)?; + } + self.transactions + .delete(rwtxn, &txid) + .map_err(DbError::from)?; + Ok(()) + } + + /// The mempool transactions that `transaction` spends from, directly or + /// through another mempool transaction. + pub fn ancestors( + &self, + rotxn: &RoTxn, + transaction: &Transaction, + ) -> Result, Error> { + let mut found = HashSet::new(); + let mut pending: VecDeque = transaction + .inputs + .iter() + .filter_map(|(outpoint, _)| parent_txid(outpoint)) + .collect(); + while let Some(txid) = pending.pop_front() { + if found.contains(&txid) { + continue; + } + let Some(tx) = self + .transactions + .try_get(rotxn, &txid) + .map_err(DbError::from)? + else { + continue; + }; + found.insert(txid); + pending.extend( + tx.transaction + .inputs + .iter() + .filter_map(|(outpoint, _)| parent_txid(outpoint)), + ); + } + Ok(found) + } + + /// The outputs this mempool holds that `transaction` spends. The confirmed + /// UTXO set holds none of them. + pub fn unconfirmed_outputs( + &self, + rotxn: &RoTxn, + transaction: &Transaction, + ) -> Result, Error> { + let mut res = HashMap::new(); + for (outpoint, _) in &transaction.inputs { + let OutPoint::Regular { txid, vout } = outpoint else { + continue; + }; + let Some(parent) = self + .transactions + .try_get(rotxn, txid) + .map_err(DbError::from)? + else { + continue; + }; + let Some(output) = + parent.transaction.outputs.as_slice().get(*vout as usize) + else { + continue; + }; + res.insert(*outpoint, output.clone()); + } + Ok(res) + } + + /// Transactions with a parent before its child. `limit` caps how many the + /// walk returns, and the result stays closed under parents, so a shorter + /// walk never gives a child whose parent it left out. The read still + /// covers the whole mempool; the limit bounds the walk and the clones. + pub fn topological( + &self, + rotxn: &RoTxn, + limit: Option, + ) -> Result, Error> { + let txs: Vec<(Txid, AuthorizedTransaction)> = self + .transactions + .iter(rotxn) + .map_err(DbError::from)? + .collect() + .map_err(DbError::from)?; + let by_txid: HashMap = + txs.iter().map(|(txid, tx)| (*txid, tx)).collect(); + let mut order = Vec::with_capacity(txs.len()); + let mut placed = HashSet::with_capacity(txs.len()); + for (txid, _) in &txs { + if limit.is_some_and(|limit| order.len() >= limit) { + break; + } + let mut stack = vec![*txid]; + while let Some(txid) = stack.last().copied() { + if limit.is_some_and(|limit| order.len() >= limit) { + break; + } + if placed.contains(&txid) { + stack.pop(); + continue; + } + let Some(tx) = by_txid.get(&txid) else { + stack.pop(); + continue; + }; + let parent = + tx.transaction.inputs.iter().find_map(|(outpoint, _)| { + parent_txid(outpoint).filter(|parent| { + by_txid.contains_key(parent) + && !placed.contains(parent) + }) + }); + match parent { + Some(parent) => stack.push(parent), + None => { + placed.insert(txid); + order.push((*tx).clone()); + stack.pop(); + } + } + } + } + Ok(order) + } + + /// The unconfirmed outputs that pay one of `addresses` and that this + /// wallet made on its own. + /// + /// `confirmed` names the outputs the wallet already holds from the chain. + /// Bitcoin Core takes an unconfirmed output only when the wallet funded + /// every input of the transaction that made it, and it walks the parents + /// to the last confirmed one. This copies that rule, so an unconfirmed + /// payment from a stranger never appears here. + /// + /// The wallet counts these outputs in its balance whatever the + /// `--spend-zero-conf-change` option says. The option decides only whether + /// the wallet may put them in a new transaction. + pub fn own_unconfirmed_utxos( + &self, + rotxn: &RoTxn, + addresses: &HashSet
, + confirmed: &HashSet, + ) -> Result, Error> { + let mut res = HashMap::new(); + // A parent comes first, so its trust and its ancestors are known by the + // time the walk reaches the child. + let mut trusted: HashSet = HashSet::new(); + let mut ancestors: HashMap> = HashMap::new(); + for tx in self.topological(rotxn, None)? { + let txid = tx.transaction.txid(); + let mut is_trusted = true; + let mut tx_ancestors = HashSet::new(); + for (outpoint, _) in &tx.transaction.inputs { + if let Some(parent) = parent_txid(outpoint) + && let Some(parent_ancestors) = ancestors.get(&parent) + { + tx_ancestors.extend(parent_ancestors.iter().copied()); + tx_ancestors.insert(parent); + } + if confirmed.contains(outpoint) || trusted.contains(outpoint) { + continue; + } + is_trusted = false; + } + ancestors.insert(txid, tx_ancestors); + if !is_trusted { + continue; + } + for (vout, output) in tx.transaction.outputs.iter().enumerate() { + if !addresses.contains(&output.address) { + continue; + } + let outpoint = OutPoint::Regular { + txid, + vout: vout as u32, + }; + trusted.insert(outpoint); + if self + .spent_utxos + .try_get(rotxn, &outpoint) + .map_err(DbError::from)? + .is_none() + { + res.insert(outpoint, output.clone()); + } + } + } + Ok(res) + } + + /// The transaction in this mempool that spends `outpoint`, if there is + /// one. + pub fn spender( + &self, + rotxn: &RoTxn, + outpoint: &OutPoint, + ) -> Result, Error> { + let txid = self + .spent_utxos + .try_get(rotxn, outpoint) + .map_err(DbError::from)?; + Ok(txid) + } + pub fn take( &self, rotxn: &RoTxn, @@ -196,10 +460,13 @@ impl MemPool { else { continue; }; + let unconfirmed = + self.unconfirmed_outputs(rwtxn, &tx.transaction)?; let targets: Vec<_> = tx .transaction .inputs .iter() + .filter(|(outpoint, _)| !unconfirmed.contains_key(outpoint)) .map(|(_, utxo_hash)| utxo_hash.into()) .collect(); match accumulator.prove(&targets) { @@ -221,3 +488,392 @@ impl MemPool { Ok(()) } } + +impl Watchable<()> for MemPool { + type WatchStream = tokio_stream::wrappers::WatchStream<()>; + + /// Get a signal that notifies whenever the mempool changes + fn watch(&self) -> Self::WatchStream { + tokio_stream::wrappers::WatchStream::new( + self.transactions.watch().clone(), + ) + } +} + +/// The mempool transaction that could have made this outpoint. A coinbase or +/// a deposit outpoint names no transaction. +fn parent_txid(outpoint: &OutPoint) -> Option { + match outpoint { + OutPoint::Regular { txid, .. } => Some(*txid), + OutPoint::Coinbase { .. } | OutPoint::Deposit(_) => None, + } +} + +#[cfg(test)] +mod test { + use bitcoin::hashes::Hash as _; + + use super::*; + use crate::types::{ + Address, OutputContent, PointedOutput, + authorization::{SigningKey, get_address}, + hash, + }; + + fn temp_env( + test_name: &str, + ) -> anyhow::Result<(temp_dir::TempDir, sneed::Env)> { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos(); + let temp_dir = temp_dir::TempDir::with_prefix(format!( + "{test_name}-{nanos}-{}", + std::process::id() + ))?; + let mut opts = heed::EnvOpenOptions::new(); + opts.map_size(16 * 1024 * 1024).max_dbs(MemPool::NUM_DBS); + let env = unsafe { sneed::Env::open(&opts, temp_dir.path()) }?; + Ok((temp_dir, env)) + } + + fn value_output(address: Address, sats: u64) -> Output { + Output { + address, + content: OutputContent::Value(bitcoin::Amount::from_sat(sats)), + } + } + + fn deposit_outpoint(seed: u8) -> OutPoint { + OutPoint::Deposit(bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([seed; 32]), + vout: 0, + }) + } + + /// Build a transaction that spends `outpoint`, worth `output` after it. + /// The mempool never checks a signature, so the authorization is empty. + fn spend( + outpoint: OutPoint, + spent: &Output, + output: Output, + ) -> AuthorizedTransaction { + let utxo_hash = hash(&PointedOutput { + outpoint, + output: spent.clone(), + }); + AuthorizedTransaction { + authorizations: Vec::new(), + transaction: Transaction { + inputs: vec![(outpoint, utxo_hash)].into(), + proof: Default::default(), + outputs: vec![output].into(), + }, + } + } + + /// A chain of `len` transactions, each spending the one before it. + fn chain( + address: Address, + start: OutPoint, + start_output: Output, + len: usize, + ) -> Vec { + let mut txs = Vec::with_capacity(len); + let mut outpoint = start; + let mut spent = start_output; + for i in 0..len { + let output = value_output(address, 10_000 - i as u64 - 1); + let tx = spend(outpoint, &spent, output.clone()); + outpoint = OutPoint::Regular { + txid: tx.transaction.txid(), + vout: 0, + }; + spent = output; + txs.push(tx); + } + txs + } + + fn owner() -> (SigningKey, Address) { + let key = SigningKey::new(&mut rand::rng()); + let address = get_address((&key).into()); + (key, address) + } + + #[test] + fn topological_puts_a_parent_before_its_child() -> anyhow::Result<()> { + let (_temp_dir, env) = temp_env("topological")?; + let mempool = MemPool::new(&env)?; + let (_key, address) = owner(); + let start = deposit_outpoint(0x01); + let start_output = value_output(address, 10_000); + let txs = chain(address, start, start_output, 4); + + let mut rwtxn = env.write_txn()?; + // Insert the children first, so key order cannot pass the test by + // accident. + for tx in txs.iter().rev() { + mempool.put(&mut rwtxn, tx)?; + } + rwtxn.commit()?; + + let rotxn = env.read_txn()?; + let order: Vec<_> = mempool + .topological(&rotxn, None)? + .into_iter() + .map(|tx| tx.transaction.txid()) + .collect(); + let expected: Vec<_> = + txs.iter().map(|tx| tx.transaction.txid()).collect(); + anyhow::ensure!( + order == expected, + "expected {expected:?}, got {order:?}" + ); + Ok(()) + } + + #[test] + fn the_mempool_refuses_a_chain_past_the_limit() -> anyhow::Result<()> { + let (_temp_dir, env) = temp_env("ancestor_limit")?; + let mempool = MemPool::new(&env)?; + let (_key, address) = owner(); + let start = deposit_outpoint(0x02); + let start_output = value_output(address, 10_000); + let txs = + chain(address, start, start_output, MAX_UNCONFIRMED_ANCESTORS + 1); + + let mut rwtxn = env.write_txn()?; + for tx in txs.iter().take(MAX_UNCONFIRMED_ANCESTORS) { + mempool.put(&mut rwtxn, tx)?; + } + let last = mempool.put(&mut rwtxn, &txs[MAX_UNCONFIRMED_ANCESTORS]); + anyhow::ensure!( + matches!(last, Err(Error::TooManyAncestors { count }) if count + == MAX_UNCONFIRMED_ANCESTORS), + "expected TooManyAncestors, got {last:?}", + ); + Ok(()) + } + + #[test] + fn a_confirmed_parent_leaves_its_child_behind() -> anyhow::Result<()> { + let (_temp_dir, env) = temp_env("delete_confirmed")?; + let mempool = MemPool::new(&env)?; + let (_key, address) = owner(); + let start = deposit_outpoint(0x03); + let start_output = value_output(address, 10_000); + let txs = chain(address, start, start_output, 2); + + let mut rwtxn = env.write_txn()?; + for tx in &txs { + mempool.put(&mut rwtxn, tx)?; + } + mempool.delete_confirmed(&mut rwtxn, txs[0].transaction.txid())?; + rwtxn.commit()?; + + let rotxn = env.read_txn()?; + let left: Vec<_> = mempool + .take_all(&rotxn)? + .into_iter() + .map(|tx| tx.transaction.txid()) + .collect(); + anyhow::ensure!( + left == vec![txs[1].transaction.txid()], + "the child must stay, got {left:?}", + ); + Ok(()) + } + + #[test] + fn a_double_spending_parent_takes_its_child() -> anyhow::Result<()> { + let (_temp_dir, env) = temp_env("delete_cascades")?; + let mempool = MemPool::new(&env)?; + let (_key, address) = owner(); + let start = deposit_outpoint(0x04); + let start_output = value_output(address, 10_000); + let txs = chain(address, start, start_output, 2); + + let mut rwtxn = env.write_txn()?; + for tx in &txs { + mempool.put(&mut rwtxn, tx)?; + } + mempool.delete(&mut rwtxn, txs[0].transaction.txid())?; + rwtxn.commit()?; + + let rotxn = env.read_txn()?; + anyhow::ensure!(mempool.take_all(&rotxn)?.is_empty()); + Ok(()) + } + + #[test] + fn own_change_is_spendable_and_a_stranger_output_is_not() + -> anyhow::Result<()> { + let (_temp_dir, env) = temp_env("trust")?; + let mempool = MemPool::new(&env)?; + let (_key, address) = owner(); + let stranger = get_address((&SigningKey::new(&mut rand::rng())).into()); + + // The wallet holds one confirmed coin and spends it. The change is + // its own, so it is trusted. + let mine = deposit_outpoint(0x05); + let mine_output = value_output(address, 10_000); + let own = spend(mine, &mine_output, value_output(address, 9_000)); + // A stranger spends a coin the wallet never held, and pays the wallet. + let theirs = deposit_outpoint(0x06); + let theirs_output = value_output(stranger, 5_000); + let gift = spend(theirs, &theirs_output, value_output(address, 4_000)); + + let mut rwtxn = env.write_txn()?; + mempool.put(&mut rwtxn, &own)?; + mempool.put(&mut rwtxn, &gift)?; + rwtxn.commit()?; + + let addresses = HashSet::from([address]); + let confirmed = HashSet::from([mine]); + let own_outpoint = OutPoint::Regular { + txid: own.transaction.txid(), + vout: 0, + }; + let gift_outpoint = OutPoint::Regular { + txid: gift.transaction.txid(), + vout: 0, + }; + + let rotxn = env.read_txn()?; + let spendable = + mempool.own_unconfirmed_utxos(&rotxn, &addresses, &confirmed)?; + anyhow::ensure!( + spendable.keys().collect::>() == vec![&own_outpoint], + "only own change is trusted, got {spendable:?}", + ); + anyhow::ensure!( + !spendable.contains_key(&gift_outpoint), + "an unconfirmed payment from someone else waits for a block", + ); + Ok(()) + } + + #[test] + fn an_untrusted_ancestor_stops_the_whole_chain() -> anyhow::Result<()> { + let (_temp_dir, env) = temp_env("trust_chain")?; + let mempool = MemPool::new(&env)?; + let (_key, address) = owner(); + let stranger = get_address((&SigningKey::new(&mut rand::rng())).into()); + + let theirs = deposit_outpoint(0x07); + let theirs_output = value_output(stranger, 5_000); + let gift_output = value_output(address, 4_000); + let gift = spend(theirs, &theirs_output, gift_output.clone()); + let gift_outpoint = OutPoint::Regular { + txid: gift.transaction.txid(), + vout: 0, + }; + let child = + spend(gift_outpoint, &gift_output, value_output(address, 3_000)); + + let mut rwtxn = env.write_txn()?; + mempool.put(&mut rwtxn, &gift)?; + mempool.put(&mut rwtxn, &child)?; + rwtxn.commit()?; + + let rotxn = env.read_txn()?; + let spendable = mempool.own_unconfirmed_utxos( + &rotxn, + &HashSet::from([address]), + &HashSet::new(), + )?; + anyhow::ensure!( + spendable.is_empty(), + "a child of a stranger's transaction is not trusted, got \ + {spendable:?}", + ); + Ok(()) + } + + /// A chain at the limit stays visible, so the balance shows the money. + /// The mempool refuses the next link, and the user reads a clear error. + #[test] + fn a_chain_at_the_limit_stays_visible() -> anyhow::Result<()> { + let (_temp_dir, env) = temp_env("ancestor_boundary")?; + let mempool = MemPool::new(&env)?; + let (_key, address) = owner(); + let start = deposit_outpoint(0x09); + let start_output = value_output(address, 10_000); + let txs = + chain(address, start, start_output, MAX_UNCONFIRMED_ANCESTORS); + + let mut rwtxn = env.write_txn()?; + for tx in &txs { + mempool.put(&mut rwtxn, tx)?; + } + rwtxn.commit()?; + + let last = &txs[MAX_UNCONFIRMED_ANCESTORS - 1]; + let tip = OutPoint::Regular { + txid: last.transaction.txid(), + vout: 0, + }; + let rotxn = env.read_txn()?; + anyhow::ensure!( + mempool.ancestors(&rotxn, &last.transaction)?.len() + == MAX_UNCONFIRMED_ANCESTORS - 1, + ); + let visible = mempool.own_unconfirmed_utxos( + &rotxn, + &HashSet::from([address]), + &HashSet::from([start]), + )?; + anyhow::ensure!( + visible.keys().collect::>() == vec![&tip], + "the money the chain holds must stay visible, got {visible:?}", + ); + drop(rotxn); + + // A child of the last link would carry 25 ancestors. + let child = spend( + tip, + &value_output(address, 10_000 - MAX_UNCONFIRMED_ANCESTORS as u64), + value_output(address, 1), + ); + let mut rwtxn = env.write_txn()?; + let refused = mempool.put(&mut rwtxn, &child); + anyhow::ensure!( + matches!(refused, Err(Error::TooManyAncestors { count }) if count + == MAX_UNCONFIRMED_ANCESTORS), + "the mempool must refuse the next link, got {refused:?}", + ); + Ok(()) + } + + #[test] + fn a_spent_unconfirmed_output_is_not_offered() -> anyhow::Result<()> { + let (_temp_dir, env) = temp_env("trust_spent")?; + let mempool = MemPool::new(&env)?; + let (_key, address) = owner(); + let start = deposit_outpoint(0x08); + let start_output = value_output(address, 10_000); + let txs = chain(address, start, start_output, 2); + + let mut rwtxn = env.write_txn()?; + for tx in &txs { + mempool.put(&mut rwtxn, tx)?; + } + rwtxn.commit()?; + + let rotxn = env.read_txn()?; + let spendable = mempool.own_unconfirmed_utxos( + &rotxn, + &HashSet::from([address]), + &HashSet::from([start]), + )?; + let tip = OutPoint::Regular { + txid: txs[1].transaction.txid(), + vout: 0, + }; + anyhow::ensure!( + spendable.keys().collect::>() == vec![&tip], + "only the last output of the chain is unspent, got {spendable:?}", + ); + Ok(()) + } +} diff --git a/lib/net/mod.rs b/lib/net/mod.rs index a1da37cf..e955656f 100644 --- a/lib/net/mod.rs +++ b/lib/net/mod.rs @@ -20,6 +20,7 @@ use tracing::instrument; use crate::{ archive::Archive, + mempool::MemPool, state::State, types::{ AuthorizedTransaction, Network, VERSION, Version, @@ -292,6 +293,7 @@ pub struct Net { pub(crate) batch_verification_ctxt: BatchVerificationContext, pub dns_resolver: Arc, magic_bytes: peer_message::MagicBytes, + mempool: MemPool, state: State, active_peers: Arc>>, // None indicates that the stream has ended @@ -410,6 +412,7 @@ impl Net { batch_verification_ctxt: self.batch_verification_ctxt, magic_bytes: self.magic_bytes, resolved_address: resolved_addr, + mempool: self.mempool.clone(), state: self.state.clone(), }; @@ -469,6 +472,7 @@ impl Net { batch_verification_ctxt: BatchVerificationContext, magic_bytes_override: Option, network: Network, + mempool: MemPool, state: State, bind_addr: SocketAddr, add_peers: HashSet, @@ -533,6 +537,7 @@ impl Net { batch_verification_ctxt, dns_resolver, magic_bytes, + mempool, state, active_peers, peer_info_tx, @@ -619,6 +624,7 @@ impl Net { batch_verification_ctxt: self.batch_verification_ctxt, magic_bytes: self.magic_bytes, resolved_address: addr.into(), + mempool: self.mempool.clone(), state: self.state.clone(), }; let (connection_handle, info_rx) = diff --git a/lib/net/peer/error.rs b/lib/net/peer/error.rs index 74c0ca6d..085d9392 100644 --- a/lib/net/peer/error.rs +++ b/lib/net/peer/error.rs @@ -557,6 +557,8 @@ pub enum Error { ReceiveResponse(#[from] connection::ReceiveResponse), #[error("Failed to push blocking task")] SendBlockingTask, + #[error("mempool error")] + MemPool(#[from] crate::mempool::Error), #[error(transparent)] SendHeartbeat(#[from] request_queue::SendHeartbeat), #[error("Failed to push info message")] @@ -574,6 +576,7 @@ impl Recoverable for Error { match self { Self::Archive(_) | Self::DbEnv(_) + | Self::MemPool(_) | Self::MissingPeerState(_) | Self::SendBlockingTask | Self::SendInfo diff --git a/lib/net/peer/mod.rs b/lib/net/peer/mod.rs index efa26d0f..10058c2a 100644 --- a/lib/net/peer/mod.rs +++ b/lib/net/peer/mod.rs @@ -18,6 +18,7 @@ use tokio::{spawn, task::JoinHandle, time::Duration}; use crate::{ archive::Archive, + mempool::MemPool, state::State, types::{ AuthorizedTransaction, Hash, Tip, Version, hash, @@ -366,6 +367,7 @@ pub struct ConnectionContext { pub batch_verification_ctxt: BatchVerificationContext, pub magic_bytes: message::MagicBytes, pub resolved_address: ResolvedPeerAddress, + pub mempool: MemPool, pub state: State, } diff --git a/lib/net/peer/task.rs b/lib/net/peer/task.rs index b6d35c78..2890fb6a 100644 --- a/lib/net/peer/task.rs +++ b/lib/net/peer/task.rs @@ -686,9 +686,12 @@ impl ConnectionTask { let txid = tx.transaction.txid(); let validate_tx_result = { let rotxn = ctxt.env.read_txn().map_err(EnvError::from)?; + let unconfirmed = + ctxt.mempool.unconfirmed_outputs(&rotxn, &tx.transaction)?; ctxt.state.validate_transaction( &rotxn, &ctxt.batch_verification_ctxt, + &unconfirmed, &tx, ) }; diff --git a/lib/node/mod.rs b/lib/node/mod.rs index 0319c17f..93efd9da 100644 --- a/lib/node/mod.rs +++ b/lib/node/mod.rs @@ -156,6 +156,7 @@ where batch_verification_ctxt, magic_bytes_override, network, + mempool.clone(), state.clone(), bind_addr, add_peers, @@ -267,13 +268,19 @@ where { { let mut rotxn = self.env.write_txn().map_err(EnvError::from)?; + let unconfirmed = self.mempool.unconfirmed_outputs( + &rotxn, + &transaction.borrow().transaction, + )?; self.state.regenerate_proof( &rotxn, + &unconfirmed, &mut transaction.borrow_mut().transaction, )?; self.state.validate_transaction( &rotxn, &self.batch_verification_ctxt, + &unconfirmed, transaction.borrow(), )?; self.mempool.put(&mut rotxn, transaction.borrow())?; @@ -322,6 +329,27 @@ where Ok(spent) } + /// What the mempool means for a wallet: the unconfirmed outputs the + /// wallet made on its own, and the confirmed outputs from `confirmed` that + /// a mempool transaction already spends. + pub fn get_mempool_view( + &self, + addresses: &HashSet
, + confirmed: &HashSet, + ) -> Result<(HashMap, HashSet), Error> { + let rotxn = self.env.read_txn().map_err(EnvError::from)?; + let unconfirmed = self + .mempool + .own_unconfirmed_utxos(&rotxn, addresses, confirmed)?; + let mut spent = HashSet::new(); + for outpoint in confirmed { + if self.mempool.spender(&rotxn, outpoint)?.is_some() { + spent.insert(*outpoint); + } + } + Ok((unconfirmed, spent)) + } + pub fn get_stxos_by_addresses( &self, addresses: &HashSet
, @@ -359,7 +387,8 @@ where pub fn regenerate_proof(&self, tx: &mut Transaction) -> Result<(), Error> { let rotxn = self.env.read_txn().map_err(EnvError::from)?; - let () = self.state.regenerate_proof(&rotxn, tx)?; + let unconfirmed = self.mempool.unconfirmed_outputs(&rotxn, tx)?; + let () = self.state.regenerate_proof(&rotxn, &unconfirmed, tx)?; Ok(()) } @@ -474,10 +503,15 @@ where ) -> Result<(Vec>, bitcoin::Amount), Error> { let mut rwtxn = self.env.write_txn().map_err(EnvError::from)?; - let transactions = self.mempool.take(&rwtxn, number)?; + // A parent comes before its child, so the block carries a chain in an + // order that validation accepts, and a child never fails because the + // walk did not reach its parent yet. + let transactions = self.mempool.topological(&rwtxn, Some(number))?; let mut fee = bitcoin::Amount::ZERO; let mut returned_transactions = vec![]; let mut spent_utxos = HashSet::new(); + // Outputs the transactions already taken for this block make. + let mut block_outputs = HashMap::::new(); for transaction in transactions { let inputs: HashSet<_> = transaction.transaction.inputs.iter().copied().collect(); @@ -492,6 +526,7 @@ where .validate_transaction( &rwtxn, &self.batch_verification_ctxt, + &block_outputs, &transaction, ) .is_err() @@ -500,9 +535,17 @@ where .delete(&mut rwtxn, transaction.transaction.txid())?; continue; } - let filled_transaction = self - .state - .fill_authorized_transaction(&rwtxn, transaction)?; + let filled_transaction = self.state.fill_authorized_transaction( + &rwtxn, + &block_outputs, + transaction, + )?; + block_outputs.extend( + filled_transaction + .transaction + .transaction + .outputs_by_outpoint(), + ); let value_in: bitcoin::Amount = filled_transaction .transaction .spent_utxos @@ -775,4 +818,9 @@ where pub fn watch_state(&self) -> impl Stream { self.state.watch() } + + /// Get a notification whenever the mempool changes + pub fn watch_mempool(&self) -> impl Stream { + self.mempool.watch() + } } diff --git a/lib/node/net_task.rs b/lib/node/net_task.rs index bd7a403b..e2e5757d 100644 --- a/lib/node/net_task.rs +++ b/lib/node/net_task.rs @@ -25,7 +25,7 @@ use tokio_stream::StreamNotifyClose; use crate::{ archive::{self, Archive}, - mempool::MemPool, + mempool::{self, MemPool}, net::{ self, Net, PeerConnectionInfo, PeerConnectionMessage, PeerInfoRx, PeerRequest, PeerResponse, PeerStateId, error::peer::Recoverable as _, @@ -88,8 +88,23 @@ fn connect_tip_( let () = archive.put_body(rwtxn, block_hash, body)?; } let () = archive.put_accumulator(rwtxn, block_hash, &accumulator)?; + // A mempool transaction that spends an input this block spends under + // another txid is a double spend, so it goes with its children. The + // block's own transactions leave their children behind, because a child of + // a confirmed parent spends a confirmed output. for transaction in &body.transactions { - let () = mempool.delete(rwtxn, transaction.txid())?; + let txid = transaction.txid(); + for (outpoint, _) in &transaction.inputs { + match mempool.spender(rwtxn, outpoint)? { + Some(spender) if spender != txid => { + let () = mempool.delete(rwtxn, spender)?; + } + Some(_) | None => (), + } + } + } + for transaction in &body.transactions { + let () = mempool.delete_confirmed(rwtxn, transaction.txid())?; } let () = mempool.regenerate_proofs(rwtxn, &accumulator)?; Ok(()) @@ -211,7 +226,7 @@ pub(in crate::node) fn disconnect_tip_( }; } for transaction in tip_body.authorized_transactions().iter().rev() { - mempool.put(rwtxn, transaction)?; + mempool.put_disconnected(rwtxn, transaction)?; } let accumulator = state.get_accumulator(rwtxn)?; mempool.regenerate_proofs(rwtxn, &accumulator)?; @@ -222,6 +237,24 @@ fn is_fatal_reorg_error(err: &Error) -> bool { !matches!(err, Error::State(_)) } +/// A peer sends a transaction this node refuses. The peer is at fault, so the +/// net task must stay up. A database error belongs to this node, and it must +/// stop the task. +fn is_fatal_peer_tx_error(err: &Error) -> bool { + match err { + // The transaction spends an output no source holds, so no proof exists + // for it. A mempool transaction that drops out leaves its children in + // that state. + Error::State(_) => false, + Error::MemPool(err) => !matches!( + err, + mempool::Error::TooManyAncestors { .. } + | mempool::Error::UtxoDoubleSpent + ), + _ => true, + } +} + /// Re-org to the specified tip, if it is better than the current tip. /// The new tip block and all ancestor blocks must exist in the node's archive. /// A result of `Ok(true)` indicates a successful re-org. @@ -1255,17 +1288,47 @@ impl NetTask { .env .write_txn() .map_err(EnvError::from)?; - let () = self.ctxt.state.regenerate_proof( - &rwtxn, - &mut new_tx.transaction, - )?; - self.ctxt.mempool.put(&mut rwtxn, &new_tx)?; - rwtxn.commit().map_err(RwTxnError::from)?; - // broadcast - let () = self - .ctxt - .net - .push_tx(HashSet::from_iter([addr]), &new_tx); + let unconfirmed = + self.ctxt.mempool.unconfirmed_outputs( + &rwtxn, + &new_tx.transaction, + )?; + let accepted = + match self.ctxt.state.regenerate_proof( + &rwtxn, + &unconfirmed, + &mut new_tx.transaction, + ) { + Ok(()) => self + .ctxt + .mempool + .put(&mut rwtxn, &new_tx) + .map_err(Error::from), + Err(err) => Err(Error::from(err)), + }; + match accepted { + Ok(()) => { + rwtxn.commit().map_err(RwTxnError::from)?; + // broadcast + let () = self.ctxt.net.push_tx( + HashSet::from_iter([addr]), + &new_tx, + ); + } + Err(err) if !is_fatal_peer_tx_error(&err) => { + // Drop the write, so the rows that `put` + // wrote before it refused never land. + drop(rwtxn); + // A peer that relays a transaction back is + // normal traffic, not a fault. + tracing::debug!( + %addr, + "this node refuses a peer's \ + transaction: {err}" + ); + } + Err(err) => return Err(err), + } } PeerConnectionInfo::Response(boxed) => { let (resp, req) = *boxed; @@ -1420,4 +1483,25 @@ mod test { fn infrastructure_error_is_fatal() { assert!(is_fatal_reorg_error(&Error::PeerInfoRxClosed)); } + + // a peer that pushes a chain past the ancestor limit, a double spend, or a + // transaction whose parent just left the mempool, must not stop the net + // task + #[test] + fn a_peer_transaction_this_node_refuses_is_not_fatal() { + use crate::{mempool, node::net_task::is_fatal_peer_tx_error}; + + assert!(!is_fatal_peer_tx_error(&Error::MemPool( + mempool::Error::TooManyAncestors { + count: mempool::MAX_UNCONFIRMED_ANCESTORS, + } + ))); + assert!(!is_fatal_peer_tx_error(&Error::MemPool( + mempool::Error::UtxoDoubleSpent + ))); + assert!(!is_fatal_peer_tx_error(&Error::State( + state::Error::NotEnoughValueIn + ))); + assert!(is_fatal_peer_tx_error(&Error::PeerInfoRxClosed)); + } } diff --git a/lib/state/block.rs b/lib/state/block.rs index 474f08dc..0e423f54 100644 --- a/lib/state/block.rs +++ b/lib/state/block.rs @@ -1,5 +1,7 @@ //! Connect and disconnect blocks +use std::collections::HashMap; + use sneed::{RoTxn, RwTxn, db::error::Error as DbError}; use crate::{ @@ -7,7 +9,7 @@ use crate::{ types::{ AccumulatorDiff, AmountOverflowError, Body, FilledTransaction, GetAddress as _, GetValue as _, Header, InPoint, MerkleRoot, OutPoint, - OutPointKey, PointedOutput, SpentOutput, UtreexoNodeHash, + OutPointKey, Output, PointedOutput, SpentOutput, UtreexoNodeHash, authorization::{self, BatchVerificationContext}, }, }; @@ -60,6 +62,10 @@ pub fn prevalidate( let mut filled_transactions: Vec = Vec::with_capacity(body.transactions.len()); let mut total_fees = bitcoin::Amount::ZERO; + // Outputs that earlier transactions in this body made. A transaction may + // spend one of them, and the accumulator holds no leaf for it, so it is + // never a proof target. + let mut body_outputs = HashMap::::new(); for transaction in &body.transactions { let txid = transaction.txid(); let mut spent_utxos = Vec::with_capacity(transaction.inputs.len()); @@ -67,13 +73,22 @@ pub fn prevalidate( Vec::::with_capacity(transaction.inputs.len()); for (outpoint, utxo_hash) in &transaction.inputs { let key = OutPointKey::from(outpoint); - let spent_output = - state.utxos.try_get(rotxn, &key)?.ok_or(error::NoUtxo { - outpoint: *outpoint, - })?; + match state.utxos.try_get(rotxn, &key)? { + Some(spent_output) => { + spent_utxos.push(spent_output); + spent_utxo_hashes.push(utxo_hash.into()); + } + None => { + let spent_output = body_outputs + .get(outpoint) + .cloned() + .ok_or(error::NoUtxo { + outpoint: *outpoint, + })?; + spent_utxos.push(spent_output); + } + } all_input_keys.push(OutPointKey::from(outpoint)); - spent_utxos.push(spent_output); - spent_utxo_hashes.push(utxo_hash.into()); accumulator_diff_txs.push((false, utxo_hash.into())); } for (vout, output) in transaction.outputs.iter().enumerate() { @@ -87,6 +102,7 @@ pub fn prevalidate( }; accumulator_diff_txs.push((true, (&pointed_output).into())); } + body_outputs.extend(transaction.outputs_by_outpoint()); if !accumulator.verify(&transaction.proof, &spent_utxo_hashes)? { return Err(Error::UtreexoProofFailed { txid }); } @@ -309,11 +325,43 @@ pub fn validate( .try_get(rotxn, &()) .map_err(DbError::from)? .unwrap_or_default(); - let filled_transactions: Vec<_> = body - .transactions - .iter() - .map(|t| state.fill_transaction(rotxn, t)) - .collect::>()?; + // Outputs that earlier transactions in this body made, and the proof + // targets each transaction keeps. An input that this body answers has no + // leaf in the accumulator, so it is never a proof target. + let mut body_outputs = HashMap::::new(); + let mut proof_targets: Vec> = + Vec::with_capacity(body.transactions.len()); + let mut filled_transactions: Vec = + Vec::with_capacity(body.transactions.len()); + for transaction in &body.transactions { + let mut spent_utxos = Vec::with_capacity(transaction.inputs.len()); + let mut spent_utxo_hashes = + Vec::::with_capacity(transaction.inputs.len()); + for (outpoint, utxo_hash) in &transaction.inputs { + let key = OutPointKey::from(outpoint); + match state.utxos.try_get(rotxn, &key)? { + Some(spent_output) => { + spent_utxos.push(spent_output); + spent_utxo_hashes.push(utxo_hash.into()); + } + None => { + let spent_output = body_outputs + .get(outpoint) + .cloned() + .ok_or(error::NoUtxo { + outpoint: *outpoint, + })?; + spent_utxos.push(spent_output); + } + } + } + body_outputs.extend(transaction.outputs_by_outpoint()); + proof_targets.push(spent_utxo_hashes); + filled_transactions.push(FilledTransaction { + spent_utxos, + transaction: transaction.clone(), + }); + } let merkle_root = Body::compute_merkle_root( &body.coinbase, filled_transactions.as_slice(), @@ -346,15 +394,12 @@ pub fn validate( // Gather all input keys to check double-spends via sort-and-scan let total_inputs = body.inputs_len(); let mut all_input_keys = Vec::with_capacity(total_inputs); - for filled_transaction in &filled_transactions { + for (filled_transaction, spent_utxo_hashes) in + filled_transactions.iter().zip(&proof_targets) + { let txid = filled_transaction.transaction.txid(); - // hashes of spent utxos, used to verify the utreexo proof - let mut spent_utxo_hashes = Vec::::with_capacity( - filled_transaction.transaction.inputs.len(), - ); for (outpoint, utxo_hash) in &filled_transaction.transaction.inputs { all_input_keys.push(OutPointKey::from(outpoint)); - spent_utxo_hashes.push(utxo_hash.into()); accumulator_diff.remove(utxo_hash.into()); } for (vout, output) in @@ -375,7 +420,7 @@ pub fn validate( .ok_or(AmountOverflowError)?; // verify utreexo proof if !accumulator - .verify(&filled_transaction.transaction.proof, &spent_utxo_hashes)? + .verify(&filled_transaction.transaction.proof, spent_utxo_hashes)? { return Err(Error::UtreexoProofFailed { txid }); } @@ -638,9 +683,14 @@ pub fn disconnect_tip( #[cfg(test)] mod test { + use std::collections::HashMap; + use thunder_types::Coinbase; - use crate::state::test::{fresh_state, value_output}; + use crate::{ + state::test::{fresh_state, value_output}, + types::FilledTransaction, + }; #[test] fn validation_rejects_outpoint_utxo_hash_mismatch() -> anyhow::Result<()> { @@ -749,7 +799,11 @@ mod test { // builds from the SUPPLIED utxo_hash. let filled = { let rotxn = env.read_txn()?; - state.fill_transaction(&rotxn, &body.transactions[0])? + state.fill_transaction( + &rotxn, + &HashMap::new(), + &body.transactions[0], + )? }; // tx validation REJECTS the outpoint/utxo_hash mismatch. @@ -793,4 +847,211 @@ mod test { } Ok(()) } + + /// Build a block that carries a parent and its child, and the state that + /// validates it. `order` names the body order. + fn chained_block( + test_name: &str, + child_first: bool, + ) -> anyhow::Result<( + temp_dir::TempDir, + sneed::Env, + crate::state::State, + crate::types::Header, + crate::types::Body, + crate::types::authorization::BatchVerificationContext, + )> { + use bitcoin::hashes::Hash as _; + + use crate::types::{ + Accumulator, AccumulatorDiff, Body, Header, OutPoint, OutPointKey, + PointedOutput, Transaction, UtreexoNodeHash, + authorization::{SigningKey, authorize, get_address}, + hash, + }; + + let (temp_dir, env, state) = fresh_state(test_name)?; + let mut rng = rand::rng(); + let batch_verification_ctxt = + crate::types::authorization::BatchVerificationContext::new( + &mut rng, + ); + let owner = SigningKey::new(&mut rng); + let owner_addr = get_address((&owner).into()); + + // One confirmed deposit funds the chain. + let deposit_outpoint = OutPoint::Deposit(bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0xCC; 32]), + vout: 0, + }); + let deposit_output = value_output(owner_addr, 10_000); + let deposit_pointed = PointedOutput { + outpoint: deposit_outpoint, + output: deposit_output.clone(), + }; + let deposit_leaf: UtreexoNodeHash = (&deposit_pointed).into(); + let deposit_hash = hash(&deposit_pointed); + + let mut accumulator = Accumulator::default(); + { + let mut diff = AccumulatorDiff::default(); + diff.insert(deposit_leaf); + accumulator.apply_diff(diff)?; + } + { + let mut rwtxn = env.write_txn()?; + state.utxos.put( + &mut rwtxn, + &OutPointKey::from(&deposit_outpoint), + &deposit_output, + )?; + state + .utreexo_accumulator + .put(&mut rwtxn, &(), &accumulator)?; + rwtxn.commit()?; + } + + // Parent spends the deposit. Child spends the parent's output. + let parent_output = value_output(owner_addr, 9_000); + let parent = Transaction { + inputs: vec![(deposit_outpoint, deposit_hash)].into(), + proof: accumulator.prove(&[deposit_leaf])?, + outputs: vec![parent_output.clone()].into(), + }; + let parent_outpoint = OutPoint::Regular { + txid: parent.txid(), + vout: 0, + }; + let parent_pointed = PointedOutput { + outpoint: parent_outpoint, + output: parent_output.clone(), + }; + let child_output = value_output(owner_addr, 8_000); + let child = Transaction { + inputs: vec![(parent_outpoint, hash(&parent_pointed))].into(), + // The accumulator holds no leaf for the parent's output, so the + // child proves nothing. + proof: accumulator.prove(&[])?, + outputs: vec![child_output.clone()].into(), + }; + + let authorized_parent = + authorize(&mut rng, &[(owner_addr, &owner)], parent)?; + let authorized_child = + authorize(&mut rng, &[(owner_addr, &owner)], child)?; + let body = if child_first { + Body::new( + vec![authorized_child, authorized_parent], + Coinbase::default(), + ) + } else { + Body::new( + vec![authorized_parent, authorized_child], + Coinbase::default(), + ) + }; + + // The parent's output is made and spent inside the block, so the + // accumulator only loses the deposit and gains the child's output. + let mut post_accumulator = Accumulator::default(); + { + let mut diff = AccumulatorDiff::default(); + diff.insert(deposit_leaf); + post_accumulator.apply_diff(diff)?; + let mut diff = AccumulatorDiff::default(); + diff.remove(deposit_leaf); + diff.insert( + (&PointedOutput { + outpoint: OutPoint::Regular { + txid: body.transactions + [if child_first { 1 } else { 0 }] + .txid(), + vout: 0, + }, + output: parent_output.clone(), + }) + .into(), + ); + diff.remove( + (&PointedOutput { + outpoint: parent_outpoint, + output: parent_output, + }) + .into(), + ); + diff.insert( + (&PointedOutput { + outpoint: OutPoint::Regular { + txid: body.transactions + [if child_first { 0 } else { 1 }] + .txid(), + vout: 0, + }, + output: child_output, + }) + .into(), + ); + post_accumulator.apply_diff(diff)?; + } + + let filled = vec![ + FilledTransaction { + spent_utxos: vec![if child_first { + parent_pointed.output.clone() + } else { + deposit_output.clone() + }], + transaction: body.transactions[0].clone(), + }, + FilledTransaction { + spent_utxos: vec![if child_first { + deposit_output + } else { + parent_pointed.output + }], + transaction: body.transactions[1].clone(), + }, + ]; + let header = Header { + merkle_root: Body::compute_merkle_root( + &body.coinbase, + filled.as_slice(), + )?, + prev_side_hash: None, + prev_main_hash: bitcoin::BlockHash::from_byte_array([0u8; 32]), + roots: post_accumulator.get_roots(), + }; + Ok((temp_dir, env, state, header, body, batch_verification_ctxt)) + } + + #[test] + fn a_block_carries_a_parent_and_its_child() -> anyhow::Result<()> { + let (_temp_dir, env, state, header, body, ctxt) = + chained_block("a_block_carries_a_parent_and_its_child", false)?; + let rotxn = env.read_txn()?; + let (fees, _) = state.validate_block(&rotxn, &ctxt, &header, &body)?; + anyhow::ensure!(fees == bitcoin::Amount::from_sat(2_000)); + let () = state + .prevalidate_block(&rotxn, &ctxt, &header, &body) + .map(|_| ())?; + Ok(()) + } + + #[test] + fn a_child_before_its_parent_is_rejected() -> anyhow::Result<()> { + let (_temp_dir, env, state, header, body, ctxt) = + chained_block("a_child_before_its_parent_is_rejected", true)?; + let rotxn = env.read_txn()?; + anyhow::ensure!( + state.validate_block(&rotxn, &ctxt, &header, &body).is_err(), + "a body that puts a child before its parent must not validate", + ); + anyhow::ensure!( + state + .prevalidate_block(&rotxn, &ctxt, &header, &body) + .is_err(), + "prevalidate must reject the same body", + ); + Ok(()) + } } diff --git a/lib/state/mod.rs b/lib/state/mod.rs index 7284c61d..d0e43921 100644 --- a/lib/state/mod.rs +++ b/lib/state/mod.rs @@ -274,16 +274,21 @@ impl State { Ok(accumulator) } - /// Regenerate utreexo proof for a tx + /// Regenerate utreexo proof for a tx. + /// + /// An input that `unconfirmed` answers has no leaf in the accumulator, so + /// it is not a proof target. The transaction that made it proves it. pub fn regenerate_proof( &self, rotxn: &RoTxn, + unconfirmed: &HashMap, tx: &mut Transaction, ) -> Result<(), Error> { let accumulator = self.get_accumulator(rotxn)?; let targets: Vec<_> = tx .inputs .iter() + .filter(|(outpoint, _)| !unconfirmed.contains_key(outpoint)) .map(|(_, utxo_hash)| utxo_hash.into()) .collect(); tx.proof = accumulator.prove(&targets)?; @@ -306,18 +311,28 @@ impl State { Ok(proof) } + /// Fill a transaction with the outputs it spends. + /// + /// `unconfirmed` holds the outputs of transactions that the chain does not + /// carry yet: the earlier transactions of a block body, or the ancestors a + /// mempool holds. Pass an empty map to read the confirmed set alone. fn fill_transaction( &self, rotxn: &RoTxn, + unconfirmed: &HashMap, transaction: &Transaction, ) -> Result { let mut spent_utxos = Vec::with_capacity(transaction.inputs.len()); for (outpoint, _) in &transaction.inputs { let key = OutPointKey::from(outpoint); - let utxo = - self.utxos.try_get(rotxn, &key)?.ok_or(error::NoUtxo { - outpoint: *outpoint, - })?; + let utxo = match self.utxos.try_get(rotxn, &key)? { + Some(utxo) => utxo, + None => { + unconfirmed.get(outpoint).cloned().ok_or(error::NoUtxo { + outpoint: *outpoint, + })? + } + }; spent_utxos.push(utxo); } Ok(FilledTransaction { @@ -329,10 +344,14 @@ impl State { pub fn fill_authorized_transaction( &self, rotxn: &RoTxn, + unconfirmed: &HashMap, transaction: AuthorizedTransaction, ) -> Result, Error> { - let filled_tx = - self.fill_transaction(rotxn, &transaction.transaction)?; + let filled_tx = self.fill_transaction( + rotxn, + unconfirmed, + &transaction.transaction, + )?; let authorizations = transaction.authorizations; Ok(Authorized { transaction: filled_tx, @@ -417,10 +436,14 @@ impl State { &self, rotxn: &RoTxn, batch_verification_ctxt: &BatchVerificationContext, + unconfirmed: &HashMap, transaction: &AuthorizedTransaction, ) -> Result { - let filled_transaction = - self.fill_transaction(rotxn, &transaction.transaction)?; + let filled_transaction = self.fill_transaction( + rotxn, + unconfirmed, + &transaction.transaction, + )?; for (authorization, spent_utxo) in transaction .authorizations .iter() diff --git a/lib/wallet/mod.rs b/lib/wallet/mod.rs index 33b849cd..e8a457c6 100644 --- a/lib/wallet/mod.rs +++ b/lib/wallet/mod.rs @@ -7,7 +7,7 @@ use bip32ish::U31; use byteorder::{BigEndian, ByteOrder}; use fallible_iterator::FallibleIterator as _; use futures::{Stream, StreamExt}; -use heed::types::{Bytes, SerdeBincode, U8}; +use heed::types::{Bytes, SerdeBincode, U8, Unit}; use sneed::{Env, EnvError, RwTxnError, UnitKey, db::error::Error as DbError}; use thiserror::Error; use tokio_stream::{StreamMap, wrappers::WatchStream}; @@ -85,6 +85,15 @@ pub struct WalletEnv; type DatabaseUnique = sneed::DatabaseUnique; type RoTxn<'a> = sneed::RoTxn<'a, heed::AnyTls, WalletEnv>; +/// The coins a wallet picked for a transaction. +pub struct SelectedCoins { + pub total: bitcoin::Amount, + pub coins: HashMap, + /// The picked coins that no block carries yet. The accumulator holds no + /// leaf for them, so a proof leaves them out. + pub unconfirmed: HashSet, +} + #[derive(Clone)] pub struct Wallet { env: sneed::Env, @@ -100,11 +109,17 @@ pub struct Wallet { DatabaseUnique, SerdeBincode
>, utxos: DatabaseUnique>, stxos: DatabaseUnique>, + /// Unconfirmed outputs that the wallet may spend. The node fills it from + /// the mempool on every sync. + unconfirmed_utxos: DatabaseUnique>, + /// Confirmed outputs that a mempool transaction already spends. Picking + /// one again would make a double spend that the mempool refuses. + mempool_spent_utxos: DatabaseUnique, _version: DatabaseUnique>, } impl Wallet { - pub const NUM_DBS: u32 = 6; + pub const NUM_DBS: u32 = 8; pub fn new(path: &Path) -> Result { std::fs::create_dir_all(path)?; @@ -152,6 +167,12 @@ impl Wallet { .map_err(EnvError::from)?; let stxos = DatabaseUnique::create(&env, &mut rwtxn, "stxos") .map_err(EnvError::from)?; + let unconfirmed_utxos = + DatabaseUnique::create(&env, &mut rwtxn, "unconfirmed_utxos") + .map_err(EnvError::from)?; + let mempool_spent_utxos = + DatabaseUnique::create(&env, &mut rwtxn, "mempool_spent_utxos") + .map_err(EnvError::from)?; let version = DatabaseUnique::create(&env, &mut rwtxn, "version") .map_err(EnvError::from)?; match version.try_get(&rwtxn, &()).map_err(DbError::from)? { @@ -181,6 +202,8 @@ impl Wallet { index_to_address, utxos, stxos, + unconfirmed_utxos, + mempool_spent_utxos, _version: version, }) } @@ -197,6 +220,12 @@ impl Wallet { .map_err(DbError::from)?; self.utxos.clear(&mut rwtxn).map_err(DbError::from)?; self.stxos.clear(&mut rwtxn).map_err(DbError::from)?; + self.unconfirmed_utxos + .clear(&mut rwtxn) + .map_err(DbError::from)?; + self.mempool_spent_utxos + .clear(&mut rwtxn) + .map_err(DbError::from)?; rwtxn.commit().map_err(RwTxnError::from)?; Ok(()) } @@ -242,6 +271,7 @@ impl Wallet { pub fn create_withdrawal( &self, accumulator: &Accumulator, + spend_zero_conf_change: bool, main_address: bitcoin::Address, value: bitcoin::Amount, main_fee: bitcoin::Amount, @@ -255,24 +285,29 @@ impl Wallet { value = %value.display_dynamic(), "Creating withdrawal" ); - let (total, coins) = self.select_coins( + let selected = self.select_coins( value .checked_add(fee) .ok_or(AmountOverflowError)? .checked_add(main_fee) .ok_or(AmountOverflowError)?, + spend_zero_conf_change, )?; - let change = total - value - fee - main_fee; + let change = selected.total - value - fee - main_fee; - let inputs: Vec<_> = coins + let inputs: Vec<_> = selected + .coins .into_iter() .map(|(outpoint, output)| { let utxo_hash = hash(&PointedOutput { outpoint, output }); (outpoint, utxo_hash) }) .collect(); - let input_utxo_hashes: Vec = - inputs.iter().map(|(_, hash)| hash.into()).collect(); + let input_utxo_hashes: Vec = inputs + .iter() + .filter(|(outpoint, _)| !selected.unconfirmed.contains(outpoint)) + .map(|(_, hash)| hash.into()) + .collect(); let proof = accumulator.prove(&input_utxo_hashes)?; let outputs = vec![ Output { @@ -299,22 +334,29 @@ impl Wallet { pub fn create_transaction( &self, accumulator: &Accumulator, + spend_zero_conf_change: bool, address: Address, value: bitcoin::Amount, fee: bitcoin::Amount, ) -> Result { - let (total, coins) = self - .select_coins(value.checked_add(fee).ok_or(AmountOverflowError)?)?; - let change = total - value - fee; - let inputs: Vec<_> = coins + let selected = self.select_coins( + value.checked_add(fee).ok_or(AmountOverflowError)?, + spend_zero_conf_change, + )?; + let change = selected.total - value - fee; + let inputs: Vec<_> = selected + .coins .into_iter() .map(|(outpoint, output)| { let utxo_hash = hash(&PointedOutput { outpoint, output }); (outpoint, utxo_hash) }) .collect(); - let input_utxo_hashes: Vec = - inputs.iter().map(|(_, hash)| hash.into()).collect(); + let input_utxo_hashes: Vec = inputs + .iter() + .filter(|(outpoint, _)| !selected.unconfirmed.contains(outpoint)) + .map(|(_, hash)| hash.into()) + .collect(); let proof = accumulator.prove(&input_utxo_hashes)?; let outputs = vec![ Output { @@ -334,23 +376,52 @@ impl Wallet { }) } + /// Pick coins worth at least `value`. A confirmed coin comes first, so a + /// chain of unconfirmed transactions only forms when the confirmed coins + /// fall short. Bitcoin Core orders its coin selection the same way. + /// + /// `spend_zero_conf_change` decides whether the wallet's own unconfirmed + /// change joins the pick at all. pub fn select_coins( &self, value: bitcoin::Amount, - ) -> Result<(bitcoin::Amount, HashMap), Error> { + spend_zero_conf_change: bool, + ) -> Result { use rayon::prelude::ParallelSliceMut; let rotxn = self.env.read_txn().map_err(EnvError::from)?; + let mempool_spent: HashSet = self + .mempool_spent_utxos + .iter_keys(&rotxn) + .map_err(DbError::from)? + .collect() + .map_err(DbError::from)?; let mut utxos: Vec<_> = self .utxos .iter(&rotxn) .map_err(DbError::from)? - .collect() + .collect::>() .map_err(DbError::from)?; + utxos.retain(|(outpoint_key, _)| !mempool_spent.contains(outpoint_key)); utxos.par_sort_unstable_by_key(|(_, output)| output.get_value()); + let mut unconfirmed_utxos: Vec<_> = if spend_zero_conf_change { + self.unconfirmed_utxos + .iter(&rotxn) + .map_err(DbError::from)? + .collect() + .map_err(DbError::from)? + } else { + Vec::new() + }; + unconfirmed_utxos + .par_sort_unstable_by_key(|(_, output)| output.get_value()); + let confirmed_count = utxos.len(); let mut selected = HashMap::new(); + let mut unconfirmed = HashSet::new(); let mut total = bitcoin::Amount::ZERO; - for (outpoint_key, output) in &utxos { + for (index, (outpoint_key, output)) in + utxos.iter().chain(&unconfirmed_utxos).enumerate() + { if output.content.is_withdrawal() { continue; } @@ -362,11 +433,18 @@ impl Wallet { .ok_or(AmountOverflowError)?; let outpoint: OutPoint = outpoint_key.into(); selected.insert(outpoint, output.clone()); + if index >= confirmed_count { + unconfirmed.insert(outpoint); + } } if total < value { return Err(Error::NotEnoughFunds); } - Ok((total, selected)) + Ok(SelectedCoins { + total, + coins: selected, + unconfirmed, + }) } pub fn delete_utxos(&self, outpoints: &[OutPoint]) -> Result<(), Error> { @@ -403,6 +481,55 @@ impl Wallet { Ok(()) } + /// Make the confirmed table say what the chain says. `utxos` is every + /// output the chain holds for this wallet, and `spent` is what a block + /// spent since the last call. + /// + /// A block that disconnects takes an output off the chain without a spend, + /// and it takes the utreexo leaf with it. A row that stays behind reads as + /// confirmed, so `create_transaction` makes it a proof target and every + /// send fails. Delete such a row here, where the chain's answer is known. + pub fn sync_confirmed( + &self, + utxos: &HashMap, + spent: &[(OutPoint, InPoint)], + ) -> Result<(), Error> { + let mut rwtxn = self.env.write_txn().map_err(EnvError::from)?; + for (outpoint, output) in utxos { + self.utxos + .put(&mut rwtxn, &OutPointKey::from(outpoint), output) + .map_err(DbError::from)?; + } + for (outpoint, inpoint) in spent { + let key = OutPointKey::from(outpoint); + let Some(output) = + self.utxos.try_get(&rwtxn, &key).map_err(DbError::from)? + else { + continue; + }; + self.utxos.delete(&mut rwtxn, &key).map_err(DbError::from)?; + let spent_output = SpentOutput { + output, + inpoint: *inpoint, + }; + self.stxos + .put(&mut rwtxn, &key, &spent_output) + .map_err(DbError::from)?; + } + let stale: Vec = self + .utxos + .iter_keys(&rwtxn) + .map_err(DbError::from)? + .filter(|key| Ok(!utxos.contains_key(&key.into()))) + .collect() + .map_err(DbError::from)?; + for key in &stale { + self.utxos.delete(&mut rwtxn, key).map_err(DbError::from)?; + } + rwtxn.commit().map_err(RwTxnError::from)?; + Ok(()) + } + pub fn put_utxos( &self, utxos: &HashMap, @@ -418,7 +545,16 @@ impl Wallet { Ok(()) } - pub fn get_balance(&self) -> Result { + /// The value the wallet holds. A confirmed output that a mempool + /// transaction already spends counts for nothing, because the money left. + /// + /// An unconfirmed output always counts toward `total` and `unconfirmed`, + /// the way Bitcoin Core always reports such value. It counts toward + /// `available` only when `spend_zero_conf_change` lets the wallet take it. + pub fn get_balance( + &self, + spend_zero_conf_change: bool, + ) -> Result { let mut balance = Balance::default(); let txn = self.env.read_txn().map_err(EnvError::from)?; let () = self @@ -426,7 +562,15 @@ impl Wallet { .iter(&txn) .map_err(DbError::from)? .map_err(|err| DbError::from(err).into()) - .for_each(|(_, utxo)| { + .for_each(|(key, utxo)| { + if self + .mempool_spent_utxos + .try_get(&txn, &key) + .map_err(DbError::from)? + .is_some() + { + return Ok(()); + } let value = utxo.get_value(); balance.total = balance .total @@ -440,9 +584,88 @@ impl Wallet { } Ok::<_, Error>(()) })?; + let () = self + .unconfirmed_utxos + .iter(&txn) + .map_err(DbError::from)? + .map_err(|err| DbError::from(err).into()) + .for_each(|(_, utxo)| { + let value = utxo.get_value(); + balance.total = balance + .total + .checked_add(value) + .ok_or(AmountOverflowError)?; + balance.unconfirmed = balance + .unconfirmed + .checked_add(value) + .ok_or(AmountOverflowError)?; + if spend_zero_conf_change && !utxo.content.is_withdrawal() { + balance.available = balance + .available + .checked_add(value) + .ok_or(AmountOverflowError)?; + } + Ok::<_, Error>(()) + })?; Ok(balance) } + /// Replace what the wallet knows about the mempool: the unconfirmed + /// outputs it may spend, and the confirmed outputs a mempool transaction + /// already spends. The node states both on every sync, so a wholesale + /// replacement leaves no stale row behind when a transaction drops out. + pub fn set_mempool_view( + &self, + unconfirmed: &HashMap, + spent: &HashSet, + ) -> Result<(), Error> { + let mut rwtxn = self.env.write_txn().map_err(EnvError::from)?; + self.unconfirmed_utxos + .clear(&mut rwtxn) + .map_err(DbError::from)?; + self.mempool_spent_utxos + .clear(&mut rwtxn) + .map_err(DbError::from)?; + for (outpoint, output) in unconfirmed { + self.unconfirmed_utxos + .put(&mut rwtxn, &OutPointKey::from(outpoint), output) + .map_err(DbError::from)?; + } + for outpoint in spent { + self.mempool_spent_utxos + .put(&mut rwtxn, &OutPointKey::from(outpoint), &()) + .map_err(DbError::from)?; + } + rwtxn.commit().map_err(RwTxnError::from)?; + Ok(()) + } + + pub fn get_mempool_spent_utxos(&self) -> Result, Error> { + let rotxn = self.env.read_txn().map_err(EnvError::from)?; + let outpoints: HashSet = self + .mempool_spent_utxos + .iter_keys(&rotxn) + .map_err(DbError::from)? + .map(|key| Ok((&key).into())) + .collect() + .map_err(DbError::from)?; + Ok(outpoints) + } + + pub fn get_unconfirmed_utxos( + &self, + ) -> Result, Error> { + let rotxn = self.env.read_txn().map_err(EnvError::from)?; + let utxos: HashMap = self + .unconfirmed_utxos + .iter(&rotxn) + .map_err(DbError::from)? + .map(|(key, output)| Ok((key.into(), output))) + .collect() + .map_err(DbError::from)?; + Ok(utxos) + } + pub fn get_utxos(&self) -> Result, Error> { let rotxn = self.env.read_txn().map_err(EnvError::from)?; let utxos: HashMap = self @@ -479,11 +702,15 @@ impl Wallet { let mut authorizations = Vec::with_capacity(transaction.inputs.len()); for (outpoint, _) in &transaction.inputs { let key = OutPointKey::from(outpoint); - let spent_utxo = self - .utxos - .try_get(&txn, &key) - .map_err(DbError::from)? - .ok_or(Error::NoUtxo)?; + let spent_utxo = + match self.utxos.try_get(&txn, &key).map_err(DbError::from)? { + Some(spent_utxo) => spent_utxo, + None => self + .unconfirmed_utxos + .try_get(&txn, &key) + .map_err(DbError::from)? + .ok_or(Error::NoUtxo)?, + }; let index = self .address_to_index .try_get(&txn, &spent_utxo.address) @@ -607,6 +834,8 @@ impl Watchable<()> for Wallet { index_to_address, utxos, stxos, + unconfirmed_utxos, + mempool_spent_utxos, _version: _, } = self; let watchables = [ @@ -615,6 +844,8 @@ impl Watchable<()> for Wallet { index_to_address.watch().clone(), utxos.watch().clone(), stxos.watch().clone(), + unconfirmed_utxos.watch().clone(), + mempool_spent_utxos.watch().clone(), ]; let streams = StreamMap::from_iter( watchables.into_iter().map(WatchStream::new).enumerate(), @@ -632,6 +863,91 @@ impl Watchable<()> for Wallet { mod tests { use super::*; + /// A block that disconnects takes an output off the chain without a + /// spend, and it takes the utreexo leaf with it. The confirmed row must go + /// too, so the output moves to the unconfirmed side, where the proof code + /// leaves it out. + #[test] + fn a_disconnected_output_leaves_the_confirmed_table() -> anyhow::Result<()> + { + use crate::types::OutputContent; + + let temp_dir = temp_dir::TempDir::with_prefix(format!( + "wallet-disconnect-{}-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos(), + std::process::id() + ))?; + let wallet = Wallet::new(temp_dir.path())?; + wallet.set_seed(&[0x99; 64])?; + let address = wallet.get_new_address()?; + let outpoint = OutPoint::Regular { + txid: crate::types::hash(&[0u8; 32]).into(), + vout: 0, + }; + let output = Output { + address, + content: OutputContent::Value(bitcoin::Amount::from_sat(10_000)), + }; + let held = HashMap::from([(outpoint, output.clone())]); + + wallet.sync_confirmed(&held, &[])?; + anyhow::ensure!(wallet.get_utxos()?.len() == 1); + + // The chain drops the output, and no block spends it. + wallet.sync_confirmed(&HashMap::new(), &[])?; + anyhow::ensure!( + wallet.get_utxos()?.is_empty(), + "the confirmed row must go when the chain drops the output", + ); + + // The transaction that made it sits in the mempool again. + wallet.set_mempool_view(&held, &HashSet::new())?; + let balance = wallet.get_balance(true)?; + anyhow::ensure!( + balance.unconfirmed == bitcoin::Amount::from_sat(10_000), + "the output reads as unconfirmed, got {balance:?}", + ); + anyhow::ensure!( + balance.total == bitcoin::Amount::from_sat(10_000), + "the wallet counts the output one time, got {balance:?}", + ); + anyhow::ensure!( + balance.available == bitcoin::Amount::from_sat(10_000), + "the wallet may take it, got {balance:?}", + ); + + // With the option off the value still shows, and the wallet may not + // take it. Bitcoin Core reports such value the same way. + let balance = wallet.get_balance(false)?; + anyhow::ensure!( + balance.unconfirmed == bitcoin::Amount::from_sat(10_000) + && balance.total == bitcoin::Amount::from_sat(10_000), + "the value stays visible, got {balance:?}", + ); + anyhow::ensure!( + balance.available == bitcoin::Amount::ZERO, + "the wallet may not take it, got {balance:?}", + ); + anyhow::ensure!( + wallet + .select_coins(bitcoin::Amount::from_sat(1_000), false) + .is_err(), + "coin selection must refuse the unconfirmed coin", + ); + + // With the option on the wallet takes it, and marks it unconfirmed so + // the proof leaves it out. + let selected = + wallet.select_coins(bitcoin::Amount::from_sat(1_000), true)?; + anyhow::ensure!( + selected.unconfirmed.contains(&outpoint), + "the wallet takes the coin and marks it unconfirmed", + ); + Ok(()) + } + #[test] fn test_get_or_generate_last_address() -> anyhow::Result<()> { let nanos = std::time::SystemTime::now() diff --git a/rpc-api/lib.rs b/rpc-api/lib.rs index 37a718e7..42d5114e 100644 --- a/rpc-api/lib.rs +++ b/rpc-api/lib.rs @@ -509,6 +509,13 @@ pub mod wallet { #[method(name = "get_wallet_utxos")] async fn get_wallet_utxos(&self) -> RpcResult>; + /// Get the unconfirmed wallet UTXOs that the wallet may spend. The + /// `--spend-unconfirmed` option decides which ones qualify. + #[method(name = "get_unconfirmed_wallet_utxos")] + async fn get_unconfirmed_wallet_utxos( + &self, + ) -> RpcResult>; + /// Attempt to mine a sidechain block #[open_api_method(output_schema(ToSchema))] #[method(name = "mine")] diff --git a/types/block/body.rs b/types/block/body.rs index bb262757..eae9df15 100644 --- a/types/block/body.rs +++ b/types/block/body.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use crate::{ + AccumulatorDiff, authorization::Authorization, block::coinbase::Coinbase, error, @@ -233,10 +234,10 @@ impl Body { where FilledTx: Borrow, { - // New leaves for the accumulator - let mut accumulator_add = Vec::::new(); - // Accumulator leaves to delete - let mut accumulator_del = Vec::::new(); + // A leaf that a transaction makes and a later transaction of the same + // body spends never reaches the accumulator, so the diff cancels the + // pair instead of asking rustreexo to add and delete it at once. + let mut diff = AccumulatorDiff::default(); for (vout, output) in coinbase_outputs.iter().enumerate() { let outpoint = OutPoint::Coinbase { txid: coinbase_txid, @@ -246,13 +247,13 @@ impl Body { outpoint, output: output.clone(), }; - accumulator_add.push((&pointed_output).into()); + diff.insert((&pointed_output).into()); } for tx in txs { let tx = tx.borrow(); let txid = tx.transaction.txid(); for (_, utxo_hash) in tx.transaction.inputs.iter() { - accumulator_del.push(utxo_hash.into()); + diff.remove(utxo_hash.into()); } for (vout, output) in tx.transaction.outputs.iter().enumerate() { let outpoint = OutPoint::Regular { @@ -263,9 +264,10 @@ impl Body { outpoint, output: output.clone(), }; - accumulator_add.push((&pointed_output).into()); + diff.insert((&pointed_output).into()); } } + let (accumulator_add, accumulator_del) = diff.into_parts(); let () = memforest .modify(&accumulator_add, &accumulator_del) .map_err(error::Utreexo)?; diff --git a/types/lib.rs b/types/lib.rs index 344008d1..214a820e 100644 --- a/types/lib.rs +++ b/types/lib.rs @@ -411,25 +411,13 @@ impl AccumulatorDiff { (self.insertions, self.deletions) } - pub fn is_empty(&self) -> bool { - self.diff.is_empty() - } -} - -#[derive(Debug, Default)] -#[repr(transparent)] -pub struct Accumulator(pub MemForest); - -impl Accumulator { - pub fn apply_diff( - &mut self, - diff: AccumulatorDiff, - ) -> Result<(), UtreexoError> { - let AccumulatorDiff { + /// Splits the diff into the insertions and the deletions, in that order. + pub fn into_parts(self) -> (Vec, Vec) { + let Self { diff, insertions: n_insertions, deletions: n_deletions, - } = diff; + } = self; let (mut insertions, mut deletions) = ( Vec::with_capacity(n_insertions), Vec::with_capacity(n_deletions), @@ -441,6 +429,24 @@ impl Accumulator { deletions.push(utxo_hash); } } + (insertions, deletions) + } + + pub fn is_empty(&self) -> bool { + self.diff.is_empty() + } +} + +#[derive(Debug, Default)] +#[repr(transparent)] +pub struct Accumulator(pub MemForest); + +impl Accumulator { + pub fn apply_diff( + &mut self, + diff: AccumulatorDiff, + ) -> Result<(), UtreexoError> { + let (insertions, deletions) = diff.into_parts(); tracing::trace!( leaves = %self.0.leaves, roots = ?self.get_roots(), @@ -657,3 +663,112 @@ mod withdrawal_bundle_order_regression { } } } + +#[cfg(test)] +mod same_block_chain { + use bitcoin::hashes::Hash as _; + + use crate::{ + Accumulator, AccumulatorDiff, Body, CoinbaseTxid, FilledTransaction, + OutPoint, Output, OutputContent, PointedOutput, Transaction, + authorization::{SigningKey, get_address}, + hash, + }; + + /// The block template and block validation must reach the same roots for + /// a block that carries a parent and its child. The template walks + /// `modify_memforest`; validation walks `AccumulatorDiff`. A leaf the + /// block makes and spends reaches neither. + #[test] + fn the_template_and_validation_agree() -> anyhow::Result<()> { + let address = get_address((&SigningKey::new(&mut rand::rng())).into()); + let value_output = |sats: u64| Output { + address, + content: OutputContent::Value(bitcoin::Amount::from_sat(sats)), + }; + + let deposit_outpoint = OutPoint::Deposit(bitcoin::OutPoint { + txid: bitcoin::Txid::from_byte_array([0xDD; 32]), + vout: 0, + }); + let deposit_output = value_output(10_000); + let deposit_pointed = PointedOutput { + outpoint: deposit_outpoint, + output: deposit_output.clone(), + }; + + let parent_output = value_output(9_000); + let parent = Transaction { + inputs: vec![(deposit_outpoint, hash(&deposit_pointed))].into(), + proof: Default::default(), + outputs: vec![parent_output.clone()].into(), + }; + let parent_outpoint = OutPoint::Regular { + txid: parent.txid(), + vout: 0, + }; + let parent_pointed = PointedOutput { + outpoint: parent_outpoint, + output: parent_output.clone(), + }; + let child_output = value_output(8_000); + let child = Transaction { + inputs: vec![(parent_outpoint, hash(&parent_pointed))].into(), + proof: Default::default(), + outputs: vec![child_output.clone()].into(), + }; + let child_pointed = PointedOutput { + outpoint: OutPoint::Regular { + txid: child.txid(), + vout: 0, + }, + output: child_output, + }; + + let seed = || -> anyhow::Result { + let mut accumulator = Accumulator::default(); + let mut diff = AccumulatorDiff::default(); + diff.insert((&deposit_pointed).into()); + accumulator.apply_diff(diff)?; + Ok(accumulator) + }; + + let mut from_template = seed()?; + // The block carries no coinbase output, so its txid never reaches a + // leaf. + let coinbase_txid = CoinbaseTxid::from(hash(&"no coinbase")); + let () = Body::modify_memforest( + coinbase_txid, + &[], + &[ + FilledTransaction { + spent_utxos: vec![deposit_output], + transaction: parent, + }, + FilledTransaction { + spent_utxos: vec![parent_output], + transaction: child, + }, + ], + &mut from_template.0, + )?; + + let mut from_validation = seed()?; + { + let mut diff = AccumulatorDiff::default(); + diff.remove((&deposit_pointed).into()); + diff.insert((&parent_pointed).into()); + diff.remove((&parent_pointed).into()); + diff.insert((&child_pointed).into()); + from_validation.apply_diff(diff)?; + } + + anyhow::ensure!( + from_template.get_roots() == from_validation.get_roots(), + "template gave {:?}, validation gave {:?}", + from_template.get_roots(), + from_validation.get_roots(), + ); + Ok(()) + } +} diff --git a/types/transaction/mod.rs b/types/transaction/mod.rs index b283a97b..ab3d1517 100644 --- a/types/transaction/mod.rs +++ b/types/transaction/mod.rs @@ -85,6 +85,22 @@ impl Transaction { borsh::object_length(self).map(|size| size as u64) } + /// The outputs this transaction creates, each with its outpoint + pub fn outputs_by_outpoint( + &self, + ) -> impl Iterator + '_ { + let txid = self.txid(); + self.outputs.iter().enumerate().map(move |(vout, output)| { + ( + OutPoint::Regular { + txid, + vout: vout as u32, + }, + output.clone(), + ) + }) + } + pub(crate) fn compute_merkle_root( &self, ) -> Result { diff --git a/types/wallet.rs b/types/wallet.rs index ab52bd83..f7f8642b 100644 --- a/types/wallet.rs +++ b/types/wallet.rs @@ -13,4 +13,13 @@ pub struct Balance { )] #[schema(value_type = u64)] pub available: Amount, + /// Value in outputs that the mempool holds and no block carries yet. It + /// always counts toward `total`. It counts toward `available` only when + /// `--spend-zero-conf-change` lets the wallet take it. + #[serde( + rename = "unconfirmed_sats", + with = "bitcoin::amount::serde::as_sat" + )] + #[schema(value_type = u64)] + pub unconfirmed: Amount, }