From a56a2f1345284d0ad331a4f4dc691475520760f5 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Fri, 9 Feb 2024 21:33:46 +0300 Subject: [PATCH] :zap: perf: use `futureunordered` for collecting results fetched from the tokio spawn tasks (#486) - using the `futureunordered` instead of vector for collecting results reduces the time it takes to fetch the results as the results do not need to come in specific order so any result that gets fetched first gets collected in the `futureunordered` type. Co-authored-by: Spencerjibz --- src/results/aggregator.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index 67fff56..dfb2489 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -8,6 +8,7 @@ use crate::models::{ engine_models::{EngineError, EngineHandler}, }; use error_stack::Report; +use futures::stream::FuturesUnordered; use regex::Regex; use reqwest::{Client, ClientBuilder}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -22,7 +23,9 @@ use tokio::task::JoinHandle; static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); /// Aliases for long type annotations -type FutureVec = Vec, Report>>>; + +type FutureVec = + FuturesUnordered, Report>>>; /// The function aggregates the scraped results from the user-selected upstream search engines. /// These engines can be chosen either from the user interface (UI) or from the configuration file. @@ -93,7 +96,7 @@ pub async fn aggregate( let mut names: Vec<&str> = Vec::with_capacity(0); // create tasks for upstream result fetching - let mut tasks: FutureVec = FutureVec::new(); + let tasks: FutureVec = FutureVec::new(); for engine_handler in upstream_search_engines { let (name, search_engine) = engine_handler.to_owned().into_name_engine();