alamin/websurfx
alamin/websurfx
0
0
Fork 0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-22 05:58:21 -05:00
Code Issues Releases Activity

✨ feat: fix bugs and add code to handle engine selections

Browse Source 
Co-authored-by: Sabrina Jewson <58880148+SabrinaJewson@users.noreply.github.com>
This commit is contained in:
neon_arch 2023-07-14 12:56:06 +03:00
parent 0781385393
commit 2f01651be0
1 changed files with 32 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/search_results_handler/aggregator.rs
View File

@ -59,26 +59,35 @@ pub async fn aggregate(
} }
// fetch results from upstream search engines simultaneously/concurrently. // fetch results from upstream search engines simultaneously/concurrently.
let search_engines: Vec<Box<dyn SearchEngine>> = upstream_search_engines let search_engines: Vec<Box<dyn SearchEngine + Send + Sync>> = upstream_search_engines
.iter() .iter()
.map(|engine| match engine.to_lowercase().as_str() { .map(|engine| match engine.to_lowercase().as_str() {
"duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box<dyn SearchEngine>, "duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box<dyn SearchEngine + Send + Sync>,
"searx " => Box::new(searx::Searx) as Box<dyn SearchEngine>, "searx" => Box::new(searx::Searx) as Box<dyn SearchEngine + Send + Sync>,
&_ => panic!("Config Error: Incorrect config file option provided"),
}) })
.collect(); .collect();
let task_capacity: usize = search_engines.len();
let tasks: Vec<JoinHandle<Result<HashMap<String, RawSearchResult>, Report<EngineError>>>> = let tasks: Vec<JoinHandle<Result<HashMap<String, RawSearchResult>, Report<EngineError>>>> =
search_engines search_engines
.iter() .into_iter()
.map(|search_engine| { .map(|search_engine| {
tokio::spawn(search_engine.results(query.clone(), page, user_agent.clone())) let query: String = query.clone();
let user_agent: String = user_agent.clone();
tokio::spawn(
async move { search_engine.results(query, page, user_agent.clone()).await },
)
}) })
.collect(); .collect();
let mut outputs = Vec::with_capacity(search_engines.len()); let mut outputs = Vec::with_capacity(task_capacity);
for task in tasks { for task in tasks {
outputs.push(task.await.ok()) if let Ok(result) = task.await {
outputs.push(result.ok())
}
} }
let mut initial: bool = true; let mut initial: bool = true;
@ -87,8 +96,7 @@ pub async fn aggregate(
if initial { if initial {
match results { match results {
Some(result) => { Some(result) => {
let new_result = result.clone(); result_map.extend(result.clone());
result_map.extend(new_result.as_ref().unwrap().clone());
counter += 1; counter += 1;
initial = false initial = false
} }
@ -105,27 +113,21 @@ pub async fn aggregate(
} else { } else {
match results { match results {
Some(result) => { Some(result) => {
let new_result = result.clone(); result.clone().into_iter().for_each(|(key, value)| {
new_result result_map
.as_ref() .entry(key)
.unwrap() .and_modify(|result| {
.clone() result.add_engines(value.clone().engine());
.into_iter() })
.for_each(|(key, value)| { .or_insert_with(|| -> RawSearchResult {
result_map RawSearchResult::new(
.entry(key) value.title.clone(),
.and_modify(|result| { value.visiting_url.clone(),
result.add_engines(value.clone().engine()); value.description.clone(),
}) value.engine.clone(),
.or_insert_with(|| -> RawSearchResult { )
RawSearchResult::new( });
value.title.clone(), });
value.visiting_url.clone(),
value.description.clone(),
value.engine.clone(),
)
});
});
counter += 1 counter += 1
} }
None => { None => {