alamin/websurfx
alamin/websurfx
0
0
Fork 0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-24 23:18:22 -05:00
Code Issues Releases Activity

optimise code for large scale server use and closes #7

Browse Source
This commit is contained in:
neon_arch 2023-05-07 21:18:19 +03:00
parent 3357fa2647
commit c5c1684cc1
3 changed files with 22 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/engines/duckduckgo.rs
View File

@ -2,9 +2,8 @@
//! by querying the upstream duckduckgo search engine with user provided query and with a page
//! number if provided.
use std::{collections::HashMap, time::Duration};
use std::collections::HashMap;
use rand::Rng;
use reqwest::header::{HeaderMap, CONTENT_TYPE, REFERER, USER_AGENT};
use scraper::{Html, Selector};
@ -47,11 +46,6 @@ pub async fn results(
}
};
// Add a random delay before making the request.
let mut rng = rand::thread_rng();
let delay_secs = rng.gen_range(1, 10);
std::thread::sleep(Duration::from_secs(delay_secs));
// initializing HeaderMap and adding appropriate headers.
let mut header_map = HeaderMap::new();
header_map.insert(USER_AGENT, user_agent.parse()?);

8
src/engines/searx.rs
View File

@ -2,10 +2,9 @@
//! by querying the upstream searx search engine instance with user provided query and with a page
//! number if provided.
use rand::Rng;
use reqwest::header::{HeaderMap, CONTENT_TYPE, REFERER, USER_AGENT};
use scraper::{Html, Selector};
use std::{collections::HashMap, time::Duration};
use std::collections::HashMap;
use crate::search_results_handler::aggregation_models::RawSearchResult;
@ -34,11 +33,6 @@ pub async fn results(
// so that upstream server recieves valid page number.
let url: String = format!("https://searx.work/search?q={query}&pageno={page}");
// Add random delay before making the request.
let mut rng = rand::thread_rng();
let delay_secs = rng.gen_range(1, 10);
std::thread::sleep(Duration::from_secs(delay_secs));
// initializing headers and adding appropriate headers.
let mut header_map = HeaderMap::new();
header_map.insert(USER_AGENT, user_agent.parse()?);

22
src/search_results_handler/aggregator.rs
View File

@ -1,7 +1,10 @@
//! This module provides the functionality to scrape and gathers all the results from the upstream
//! search engines and then removes duplicate results.
use std::collections::HashMap;
use std::{collections::HashMap, time::Duration};
use rand::Rng;
use tokio::join;
use super::{
aggregation_models::{RawSearchResult, SearchResult, SearchResults},
@ -39,10 +42,19 @@ pub async fn aggregate(
let user_agent: String = random_user_agent();
let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
let ddg_map_results: HashMap<String, RawSearchResult> =
duckduckgo::results(query, page, &user_agent).await?;
let searx_map_results: HashMap<String, RawSearchResult> =
searx::results(query, page, &user_agent).await?;
// Add a random delay before making the request.
let mut rng = rand::thread_rng();
let delay_secs = rng.gen_range(1, 10);
std::thread::sleep(Duration::from_secs(delay_secs));
// fetch results from upstream search engines simultaneously/concurrently.
let (ddg_map_results, searx_map_results) = join!(
duckduckgo::results(query, page, &user_agent),
searx::results(query, page, &user_agent)
);
let ddg_map_results: HashMap<String, RawSearchResult> = ddg_map_results?;
let searx_map_results: HashMap<String, RawSearchResult> = searx_map_results?;
result_map.extend(ddg_map_results);