alamin/websurfx
alamin/websurfx
0
0
Fork 0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-23 06:28:23 -05:00
Code Issues Releases Activity

⚡ perf: use futureunordered for collecting results fetched from the tokio spawn tasks (#486)

Browse Source 
- using the `futureunordered` instead of vector for collecting results
  reduces the time it takes to fetch the results as the results do not
  need to come in specific order so any result that gets fetched first
  gets collected in the `futureunordered` type.

Co-authored-by: Spencerjibz <spencernajib2@gmail.com>
This commit is contained in:
neon_arch 2024-02-09 21:33:46 +03:00
parent eceaf15b2e
commit a56a2f1345
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/results/aggregator.rs
View File

@ -8,6 +8,7 @@ use crate::models::{
engine_models::{EngineError, EngineHandler}, engine_models::{EngineError, EngineHandler},
}; };
use error_stack::Report; use error_stack::Report;
use futures::stream::FuturesUnordered;
use regex::Regex; use regex::Regex;
use reqwest::{Client, ClientBuilder}; use reqwest::{Client, ClientBuilder};
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
@ -22,7 +23,9 @@ use tokio::task::JoinHandle;
static CLIENT: std::sync::OnceLock<Client> = std::sync::OnceLock::new(); static CLIENT: std::sync::OnceLock<Client> = std::sync::OnceLock::new();
/// Aliases for long type annotations /// Aliases for long type annotations
type FutureVec = Vec<JoinHandle<Result<Vec<(String, SearchResult)>, Report<EngineError>>>>;
type FutureVec =
FuturesUnordered<JoinHandle<Result<Vec<(String, SearchResult)>, Report<EngineError>>>>;
/// The function aggregates the scraped results from the user-selected upstream search engines. /// The function aggregates the scraped results from the user-selected upstream search engines.
/// These engines can be chosen either from the user interface (UI) or from the configuration file. /// These engines can be chosen either from the user interface (UI) or from the configuration file.
@ -93,7 +96,7 @@ pub async fn aggregate(
let mut names: Vec<&str> = Vec::with_capacity(0); let mut names: Vec<&str> = Vec::with_capacity(0);
// create tasks for upstream result fetching // create tasks for upstream result fetching
let mut tasks: FutureVec = FutureVec::new(); let tasks: FutureVec = FutureVec::new();
for engine_handler in upstream_search_engines { for engine_handler in upstream_search_engines {
let (name, search_engine) = engine_handler.to_owned().into_name_engine(); let (name, search_engine) = engine_handler.to_owned().into_name_engine();