1use std::{
29 collections::{HashMap, HashSet},
30 net::{IpAddr, SocketAddr},
31 pin::Pin,
32 task::{Context, Poll},
33 time::Duration,
34};
35
36use futures::{
37 future::TryFutureExt,
38 ready,
39 stream::{FuturesUnordered, Stream},
40 FutureExt,
41};
42use pin_project::{pin_project, pinned_drop};
43use thiserror::Error;
44use tokio::{sync::oneshot, task::JoinHandle};
45use tower::{Service, ServiceExt};
46use tracing_futures::Instrument;
47
48use zebra_chain::{
49 block::Height,
50 parameters::NetworkUpgrade,
51 transaction::{self, UnminedTxId, VerifiedUnminedTx},
52 transparent,
53};
54use zebra_consensus::transaction as tx;
55use zebra_network::{self as zn, PeerSocketAddr};
56use zebra_node_services::mempool::Gossip;
57use zebra_state::{self as zs, CloneError};
58
59use crate::components::{
60 mempool::crawler::RATE_LIMIT_DELAY,
61 sync::{BLOCK_DOWNLOAD_TIMEOUT, BLOCK_VERIFY_TIMEOUT},
62};
63
64use super::MempoolError;
65
66type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
67
68pub(crate) const TRANSACTION_DOWNLOAD_TIMEOUT: Duration = BLOCK_DOWNLOAD_TIMEOUT;
74
75pub(crate) const TRANSACTION_VERIFY_TIMEOUT: Duration = BLOCK_VERIFY_TIMEOUT;
83
84pub const MAX_INBOUND_CONCURRENCY: usize = 500;
108
109pub const MAX_INBOUND_CONCURRENCY_PER_PEER: usize = 5;
118
119#[derive(Copy, Clone, Debug, Eq, PartialEq)]
121struct CancelDownloadAndVerify;
122
123#[derive(Error, Debug, Clone)]
125#[allow(dead_code)]
126pub enum TransactionDownloadVerifyError {
127 #[error("transaction is already in state")]
128 InState,
129
130 #[error("error in state service: {0}")]
131 StateError(#[source] CloneError),
132
133 #[error("error downloading transaction: {0}")]
134 DownloadFailed(#[source] CloneError),
135
136 #[error("transaction download / verification was cancelled")]
137 Cancelled,
138
139 #[error("transaction did not pass consensus validation: {error}")]
140 Invalid {
141 error: zebra_consensus::error::TransactionError,
142 advertiser_addr: Option<PeerSocketAddr>,
143 transaction_upgrade: Option<NetworkUpgrade>,
145 verification_height: Height,
147 },
148}
149
150#[pin_project(PinnedDrop)]
152#[derive(Debug)]
153pub struct Downloads<ZN, ZV, ZS>
154where
155 ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
156 ZN::Future: Send,
157 ZV: Service<tx::MempoolRequest, Response = tx::MempoolResponse, Error = BoxError>
158 + Send
159 + Clone
160 + 'static,
161 ZV::Future: Send,
162 ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
163 ZS::Future: Send,
164{
165 network: ZN,
169
170 verifier: ZV,
172
173 state: ZS,
175
176 #[pin]
179 pending: FuturesUnordered<
180 JoinHandle<
181 Result<
182 Result<
183 (
184 VerifiedUnminedTx,
185 Vec<transparent::OutPoint>,
186 Option<Height>,
187 Option<oneshot::Sender<Result<(), BoxError>>>,
188 ),
189 Box<(TransactionDownloadVerifyError, UnminedTxId)>,
190 >,
191 (UnminedTxId, tokio::time::error::Elapsed),
192 >,
193 >,
194 >,
195
196 cancel_handles: HashMap<
201 UnminedTxId,
202 (
203 oneshot::Sender<CancelDownloadAndVerify>,
204 Gossip,
205 Option<SocketAddr>,
206 ),
207 >,
208
209 pending_per_peer: HashMap<IpAddr, usize>,
222}
223
224impl<ZN, ZV, ZS> Stream for Downloads<ZN, ZV, ZS>
225where
226 ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
227 ZN::Future: Send,
228 ZV: Service<tx::MempoolRequest, Response = tx::MempoolResponse, Error = BoxError>
229 + Send
230 + Clone
231 + 'static,
232 ZV::Future: Send,
233 ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
234 ZS::Future: Send,
235{
236 type Item = Result<
237 Result<
238 (
239 VerifiedUnminedTx,
240 Vec<transparent::OutPoint>,
241 Option<Height>,
242 Option<oneshot::Sender<Result<(), BoxError>>>,
243 ),
244 Box<(UnminedTxId, TransactionDownloadVerifyError)>,
245 >,
246 (UnminedTxId, tokio::time::error::Elapsed),
247 >;
248
249 fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
250 let this = self.project();
251 let item = if let Some(join_result) = ready!(this.pending.poll_next(cx)) {
261 let result = join_result.expect("transaction download and verify tasks must not panic");
262 let (result, completed_txid) = match result {
263 Ok(Ok((tx, spent_mempool_outpoints, tip_height, rsp_tx))) => {
264 let hash = tx.transaction.id;
265 (
266 Ok(Ok((tx, spent_mempool_outpoints, tip_height, rsp_tx))),
267 Some(hash),
268 )
269 }
270 Ok(Err(boxed_err)) => {
271 let (e, hash) = *boxed_err;
272 (Ok(Err(Box::new((hash, e)))), Some(hash))
273 }
274 Err((txid, elapsed)) => {
275 (Err((txid, elapsed)), Some(txid))
284 }
285 };
286
287 if let Some(hash) = completed_txid {
288 if let Some((_, _gossip, Some(source))) = this.cancel_handles.remove(&hash) {
289 Self::release_peer_slot(this.pending_per_peer, source);
290 }
291 }
292
293 Some(result)
294 } else {
295 None
296 };
297
298 Poll::Ready(item)
299 }
300
301 fn size_hint(&self) -> (usize, Option<usize>) {
302 self.pending.size_hint()
303 }
304}
305
306impl<ZN, ZV, ZS> Downloads<ZN, ZV, ZS>
307where
308 ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
309 ZN::Future: Send,
310 ZV: Service<tx::MempoolRequest, Response = tx::MempoolResponse, Error = BoxError>
311 + Send
312 + Clone
313 + 'static,
314 ZV::Future: Send,
315 ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
316 ZS::Future: Send,
317{
318 pub fn new(network: ZN, verifier: ZV, state: ZS) -> Self {
328 Self {
329 network,
330 verifier,
331 state,
332 pending: FuturesUnordered::new(),
333 cancel_handles: HashMap::new(),
334 pending_per_peer: HashMap::new(),
335 }
336 }
337
338 #[instrument(skip(self, gossiped_tx), fields(txid = %gossiped_tx.id()))]
346 #[allow(clippy::unwrap_in_result)]
347 pub fn download_if_needed_and_verify(
348 &mut self,
349 gossiped_tx: Gossip,
350 source: Option<SocketAddr>,
351 mut rsp_tx: Option<oneshot::Sender<Result<(), BoxError>>>,
352 ) -> Result<(), MempoolError> {
353 let txid = gossiped_tx.id();
354
355 if self.cancel_handles.contains_key(&txid) {
356 debug!(
357 ?txid,
358 queue_len = self.pending.len(),
359 ?MAX_INBOUND_CONCURRENCY,
360 "transaction id already queued for inbound download: ignored transaction"
361 );
362 metrics::gauge!("mempool.currently.queued.transactions",)
363 .set(self.pending.len() as f64);
364
365 return Err(MempoolError::AlreadyQueued);
366 }
367
368 if self.pending.len() >= MAX_INBOUND_CONCURRENCY {
369 debug!(
370 ?txid,
371 queue_len = self.pending.len(),
372 ?MAX_INBOUND_CONCURRENCY,
373 "too many transactions queued for inbound download: ignored transaction"
374 );
375 metrics::gauge!("mempool.currently.queued.transactions",)
376 .set(self.pending.len() as f64);
377
378 return Err(MempoolError::FullQueue);
379 }
380
381 if let Some(source) = source {
384 let count = self
385 .pending_per_peer
386 .get(&source.ip())
387 .copied()
388 .unwrap_or(0);
389 if count >= MAX_INBOUND_CONCURRENCY_PER_PEER {
390 debug!(
391 ?txid,
392 peer_queue_len = count,
393 ?MAX_INBOUND_CONCURRENCY_PER_PEER,
394 "too many transactions queued for this peer: ignored transaction"
395 );
396 metrics::counter!("mempool.full_queue.per_peer.total").increment(1);
397 return Err(MempoolError::FullQueue);
398 }
399 }
400
401 let (cancel_tx, mut cancel_rx) = oneshot::channel::<CancelDownloadAndVerify>();
403
404 let network = self.network.clone();
405 let verifier = self.verifier.clone();
406 let mut state = self.state.clone();
407 let pushed_advertiser_addr = source.map(PeerSocketAddr::from);
408
409 let gossiped_tx_req = gossiped_tx.clone();
410
411 let fut = async move {
412 Self::transaction_in_best_chain(&mut state, txid).await?;
414
415 trace!(?txid, "transaction is not in best chain");
416
417 let (tip_height, next_height) = match state.oneshot(zs::Request::Tip).await {
418 Ok(zs::Response::Tip(None)) => Ok((None, Height(0))),
419 Ok(zs::Response::Tip(Some((height, _hash)))) => {
420 let next_height =
421 (height + 1).expect("valid heights are far below the maximum");
422 Ok((Some(height), next_height))
423 }
424 Ok(_) => unreachable!("wrong response"),
425 Err(e) => Err(TransactionDownloadVerifyError::StateError(e.into())),
426 }?;
427
428 trace!(?txid, ?next_height, "got next height");
429
430 let (tx, advertiser_addr) = match gossiped_tx {
431 Gossip::Id(txid) => {
432 let req = zn::Request::TransactionsById(std::iter::once(txid).collect());
433
434 let tx = match network
435 .oneshot(req)
436 .await
437 .map_err(CloneError::from)
438 .map_err(TransactionDownloadVerifyError::DownloadFailed)?
439 {
440 zn::Response::Transactions(mut txs) => txs.pop().ok_or_else(|| {
441 TransactionDownloadVerifyError::DownloadFailed(
442 BoxError::from("no transactions returned").into(),
443 )
444 })?,
445 _ => unreachable!("wrong response to transaction request"),
446 };
447
448 let (tx, advertiser_addr) = tx.available().expect(
449 "unexpected missing tx status: single tx failures should be errors",
450 );
451
452 metrics::counter!(
453 "mempool.downloaded.transactions.total",
454 "version" => format!("{}",tx.transaction.version()),
455 ).increment(1);
456 (tx, advertiser_addr)
457 }
458 Gossip::Tx(tx) => {
459 metrics::counter!(
460 "mempool.pushed.transactions.total",
461 "version" => format!("{}",tx.transaction.version()),
462 ).increment(1);
463 (tx, pushed_advertiser_addr)
464 }
465 };
466
467 trace!(?txid, "got tx");
468
469 let transaction_upgrade = tx.transaction.network_upgrade();
470 let result = verifier
471 .oneshot(tx::MempoolRequest {
472 transaction: tx.clone(),
473 height: next_height,
474 })
475 .map_ok(|rsp| {
476 let tx::MempoolResponse { transaction, spent_mempool_outpoints } = rsp;
477
478 (transaction, spent_mempool_outpoints, tip_height)
479 })
480 .await;
481
482 trace!(?txid, result = ?result.as_ref().map(|_tx| ()), "verified transaction for the mempool");
484
485 result.map_err(|error| TransactionDownloadVerifyError::Invalid {
486 error: error.into(),
487 advertiser_addr,
488 transaction_upgrade,
489 verification_height: next_height,
490 })
491 }
492 .map_ok(|(tx, spent_mempool_outpoints, tip_height)| {
493 metrics::counter!(
494 "mempool.verified.transactions.total",
495 "version" => format!("{}", tx.transaction.transaction.version()),
496 ).increment(1);
497 (tx, spent_mempool_outpoints, tip_height)
498 })
499 .map_err(move |e| Box::new((e, txid)))
502 .inspect(move |result| {
503 let result = result.as_ref().map(|_tx| txid);
505 debug!("mempool transaction result: {result:?}");
506 })
507 .in_current_span();
508
509 let task = tokio::spawn(async move {
510 let fut = tokio::time::timeout(RATE_LIMIT_DELAY, fut);
511
512 let result = tokio::select! {
514 biased;
515 _ = &mut cancel_rx => {
516 trace!("task cancelled prior to completion");
517 metrics::counter!("mempool.cancelled.verify.tasks.total").increment(1);
518 if let Some(rsp_tx) = rsp_tx.take() {
519 let _ = rsp_tx.send(Err("verification cancelled".into()));
520 }
521
522 Ok(Err(Box::new((TransactionDownloadVerifyError::Cancelled, txid))))
523 }
524 verification = fut => {
525 verification
526 .inspect_err(|_elapsed| {
527 if let Some(rsp_tx) = rsp_tx.take() {
528 let _ = rsp_tx.send(Err("timeout waiting for verification result".into()));
529 }
530 })
531 .map_err(|elapsed| (txid, elapsed))
532 .map(|inner_result| {
533 match inner_result {
534 Ok((transaction, spent_mempool_outpoints, tip_height)) => Ok((transaction, spent_mempool_outpoints, tip_height, rsp_tx)),
535 Err(boxed_err) => {
536 let (tx_verifier_error, tx_id) = *boxed_err;
537 if let Some(rsp_tx) = rsp_tx.take() {
538 let error_msg = format!(
539 "failed to validate tx: {tx_id}, error: {tx_verifier_error}"
540 );
541 let _ = rsp_tx.send(Err(error_msg.into()));
542 };
543
544 Err(Box::new((tx_verifier_error, tx_id)))
545 }
546 }
547 })
548 },
549 };
550
551 result
552 });
553
554 self.pending.push(task);
555 assert!(
556 self.cancel_handles
557 .insert(txid, (cancel_tx, gossiped_tx_req, source))
558 .is_none(),
559 "transactions are only queued once"
560 );
561 if let Some(source) = source {
562 *self.pending_per_peer.entry(source.ip()).or_insert(0) += 1;
565 }
566
567 debug!(
568 ?txid,
569 queue_len = self.pending.len(),
570 ?MAX_INBOUND_CONCURRENCY,
571 "queued transaction hash for download"
572 );
573 metrics::gauge!("mempool.currently.queued.transactions",).set(self.pending.len() as f64);
574 metrics::counter!("mempool.queued.transactions.total").increment(1);
575
576 Ok(())
577 }
578
579 pub fn cancel(&mut self, mined_ids: &HashSet<transaction::Hash>) {
582 let removed_txids: Vec<UnminedTxId> = self
585 .cancel_handles
586 .keys()
587 .filter(|txid| mined_ids.contains(&txid.mined_id()))
588 .cloned()
589 .collect();
590
591 for txid in removed_txids {
592 if let Some((cancel_tx, _gossip, source)) = self.cancel_handles.remove(&txid) {
593 let _ = cancel_tx.send(CancelDownloadAndVerify);
594 if let Some(source) = source {
595 Self::release_peer_slot(&mut self.pending_per_peer, source);
596 }
597 }
598 }
599 }
600
601 pub fn cancel_all(&mut self) {
604 let _ = std::mem::take(&mut self.pending);
606 for (_hash, (cancel_tx, _gossip, _source)) in self.cancel_handles.drain() {
610 let _ = cancel_tx.send(CancelDownloadAndVerify);
611 }
612 self.pending_per_peer.clear();
613 assert!(self.pending.is_empty());
614 assert!(self.cancel_handles.is_empty());
615 metrics::gauge!("mempool.currently.queued.transactions",).set(self.pending.len() as f64);
616 }
617
618 fn release_peer_slot(pending_per_peer: &mut HashMap<IpAddr, usize>, source: SocketAddr) {
621 let ip = source.ip();
622 if let Some(count) = pending_per_peer.get_mut(&ip) {
623 *count = count.saturating_sub(1);
624 if *count == 0 {
625 pending_per_peer.remove(&ip);
626 }
627 }
628 }
629
630 #[allow(dead_code)]
632 pub fn in_flight(&self) -> usize {
633 self.pending.len()
634 }
635
636 pub fn transaction_requests(&self) -> impl Iterator<Item = &Gossip> {
638 self.cancel_handles
639 .iter()
640 .map(|(_tx_id, (_handle, tx, _source))| tx)
641 }
642
643 async fn transaction_in_best_chain(
645 state: &mut ZS,
646 txid: UnminedTxId,
647 ) -> Result<(), TransactionDownloadVerifyError> {
648 match state
649 .ready()
650 .await
651 .map_err(CloneError::from)
652 .map_err(TransactionDownloadVerifyError::StateError)?
653 .call(zs::Request::Transaction(txid.mined_id()))
654 .await
655 {
656 Ok(zs::Response::Transaction(None)) => Ok(()),
657 Ok(zs::Response::Transaction(Some(_))) => Err(TransactionDownloadVerifyError::InState),
658 Ok(_) => unreachable!("wrong response"),
659 Err(e) => Err(TransactionDownloadVerifyError::StateError(e.into())),
660 }?;
661
662 Ok(())
663 }
664}
665
666#[pinned_drop]
667impl<ZN, ZV, ZS> PinnedDrop for Downloads<ZN, ZV, ZS>
668where
669 ZN: Service<zn::Request, Response = zn::Response, Error = BoxError> + Send + Clone + 'static,
670 ZN::Future: Send,
671 ZV: Service<tx::MempoolRequest, Response = tx::MempoolResponse, Error = BoxError>
672 + Send
673 + Clone
674 + 'static,
675 ZV::Future: Send,
676 ZS: Service<zs::Request, Response = zs::Response, Error = BoxError> + Send + Clone + 'static,
677 ZS::Future: Send,
678{
679 fn drop(mut self: Pin<&mut Self>) {
680 self.cancel_all();
681
682 metrics::gauge!("mempool.currently.queued.transactions").set(0 as f64);
683 }
684}
685
686#[cfg(test)]
687mod tests;