alamin/websurfx
alamin/websurfx
0
0
Fork 0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-22 05:58:21 -05:00
Code Issues Releases Activity

✨ feat: add code to pass engine errors to the UI

Browse Source
This commit is contained in:
neon_arch 2023-07-14 21:27:23 +03:00
parent 3aeb3b3e73
commit a3edf70d1e
1 changed files with 29 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
src/search_results_handler/aggregator.rs
View File

@ -8,7 +8,7 @@ use rand::Rng;
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use super::{ use super::{
aggregation_models::{RawSearchResult, SearchResult, SearchResults}, aggregation_models::{EngineErrorInfo, RawSearchResult, SearchResult, SearchResults},
user_agent::random_user_agent, user_agent::random_user_agent,
}; };
@ -18,6 +18,8 @@ use crate::engines::{
searx, searx,
}; };
type FutureVec = Vec<JoinHandle<Result<HashMap<String, RawSearchResult>, Report<EngineError>>>>;
/// A function that aggregates all the scraped results from the above upstream engines and /// A function that aggregates all the scraped results from the above upstream engines and
/// then removes duplicate results and if two results are found to be from two or more engines /// then removes duplicate results and if two results are found to be from two or more engines
/// then puts their names together to show the results are fetched from these upstream engines /// then puts their names together to show the results are fetched from these upstream engines
@ -70,49 +72,48 @@ pub async fn aggregate(
let task_capacity: usize = search_engines.len(); let task_capacity: usize = search_engines.len();
let tasks: Vec<JoinHandle<Result<HashMap<String, RawSearchResult>, Report<EngineError>>>> = let tasks: FutureVec = search_engines
search_engines .into_iter()
.into_iter() .map(|search_engine| {
.map(|search_engine| { let query: String = query.clone();
let query: String = query.clone(); let user_agent: String = user_agent.clone();
let user_agent: String = user_agent.clone(); tokio::spawn(
tokio::spawn( async move { search_engine.results(query, page, user_agent.clone()).await },
async move { search_engine.results(query, page, user_agent.clone()).await }, )
) })
}) .collect();
.collect();
let mut outputs = Vec::with_capacity(task_capacity); let mut outputs = Vec::with_capacity(task_capacity);
for task in tasks { for task in tasks {
if let Ok(result) = task.await { if let Ok(result) = task.await {
outputs.push(result.ok()) outputs.push(result)
} }
} }
let mut engine_errors_info: Vec<EngineErrorInfo> = Vec::new();
let mut initial: bool = true; let mut initial: bool = true;
let mut counter: usize = 0; let mut counter: usize = 0;
outputs.iter().for_each(|results| { outputs.iter().for_each(|results| {
if initial { if initial {
match results { match results {
Some(result) => { Ok(result) => {
result_map.extend(result.clone()); result_map.extend(result.clone());
counter += 1; counter += 1;
initial = false initial = false
} }
None => { Err(error_type) => {
if debug { engine_errors_info.push(EngineErrorInfo::new(
log::error!( error_type.downcast_ref::<EngineError>().unwrap(),
"Error fetching results from {}", upstream_search_engines[counter].clone(),
upstream_search_engines[counter] ));
);
};
counter += 1 counter += 1
} }
} }
} else { } else {
match results { match results {
Some(result) => { Ok(result) => {
result.clone().into_iter().for_each(|(key, value)| { result.clone().into_iter().for_each(|(key, value)| {
result_map result_map
.entry(key) .entry(key)
@ -130,13 +131,11 @@ pub async fn aggregate(
}); });
counter += 1 counter += 1
} }
None => { Err(error_type) => {
if debug { engine_errors_info.push(EngineErrorInfo::new(
log::error!( error_type.downcast_ref::<EngineError>().unwrap(),
"Error fetching results from {}", upstream_search_engines[counter].clone(),
upstream_search_engines[counter] ));
);
};
counter += 1 counter += 1
} }
} }
@ -157,5 +156,6 @@ pub async fn aggregate(
}) })
.collect(), .collect(),
query.to_string(), query.to_string(),
engine_errors_info,
)) ))
} }