From c5c1684cc1cbfe40a89b80194335baa68237ffe1 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 7 May 2023 21:18:19 +0300 Subject: [PATCH] optimise code for large scale server use and closes #7 --- src/engines/duckduckgo.rs | 8 +------ src/engines/searx.rs | 8 +------ src/search_results_handler/aggregator.rs | 28 +++++++++++++++++------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index d1c5bd0..fe6c370 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -2,9 +2,8 @@ //! by querying the upstream duckduckgo search engine with user provided query and with a page //! number if provided. -use std::{collections::HashMap, time::Duration}; +use std::collections::HashMap; -use rand::Rng; use reqwest::header::{HeaderMap, CONTENT_TYPE, REFERER, USER_AGENT}; use scraper::{Html, Selector}; @@ -47,11 +46,6 @@ pub async fn results( } }; - // Add a random delay before making the request. - let mut rng = rand::thread_rng(); - let delay_secs = rng.gen_range(1, 10); - std::thread::sleep(Duration::from_secs(delay_secs)); - // initializing HeaderMap and adding appropriate headers. let mut header_map = HeaderMap::new(); header_map.insert(USER_AGENT, user_agent.parse()?); diff --git a/src/engines/searx.rs b/src/engines/searx.rs index 508655c..afb34cb 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -2,10 +2,9 @@ //! by querying the upstream searx search engine instance with user provided query and with a page //! number if provided. -use rand::Rng; use reqwest::header::{HeaderMap, CONTENT_TYPE, REFERER, USER_AGENT}; use scraper::{Html, Selector}; -use std::{collections::HashMap, time::Duration}; +use std::collections::HashMap; use crate::search_results_handler::aggregation_models::RawSearchResult; @@ -34,11 +33,6 @@ pub async fn results( // so that upstream server recieves valid page number. let url: String = format!("https://searx.work/search?q={query}&pageno={page}"); - // Add random delay before making the request. - let mut rng = rand::thread_rng(); - let delay_secs = rng.gen_range(1, 10); - std::thread::sleep(Duration::from_secs(delay_secs)); - // initializing headers and adding appropriate headers. let mut header_map = HeaderMap::new(); header_map.insert(USER_AGENT, user_agent.parse()?); diff --git a/src/search_results_handler/aggregator.rs b/src/search_results_handler/aggregator.rs index 5fd5770..e4384fd 100644 --- a/src/search_results_handler/aggregator.rs +++ b/src/search_results_handler/aggregator.rs @@ -1,7 +1,10 @@ //! This module provides the functionality to scrape and gathers all the results from the upstream //! search engines and then removes duplicate results. -use std::collections::HashMap; +use std::{collections::HashMap, time::Duration}; + +use rand::Rng; +use tokio::join; use super::{ aggregation_models::{RawSearchResult, SearchResult, SearchResults}, @@ -14,14 +17,14 @@ use crate::engines::{duckduckgo, searx}; /// then removes duplicate results and if two results are found to be from two or more engines /// then puts their names together to show the results are fetched from these upstream engines /// and then removes all data from the HashMap and puts into a struct of all results aggregated -/// into a vector and also adds the query used into the struct this is neccessory because +/// into a vector and also adds the query used into the struct this is neccessory because /// otherwise the search bar in search remains empty if searched from the query url /// /// # Example: /// /// If you search from the url like `https://127.0.0.1/search?q=huston` then the search bar should /// contain the word huston and not remain empty. -/// +/// /// # Arguments /// /// * `query` - Accepts a string to query with the above upstream search engines. @@ -29,7 +32,7 @@ use crate::engines::{duckduckgo, searx}; /// /// # Error /// -/// Returns an error a reqwest and scraping selector errors if any error occurs in the results +/// Returns an error a reqwest and scraping selector errors if any error occurs in the results /// function in either `searx` or `duckduckgo` or both otherwise returns a `SearchResults struct` /// containing appropriate values. pub async fn aggregate( @@ -39,10 +42,19 @@ pub async fn aggregate( let user_agent: String = random_user_agent(); let mut result_map: HashMap = HashMap::new(); - let ddg_map_results: HashMap = - duckduckgo::results(query, page, &user_agent).await?; - let searx_map_results: HashMap = - searx::results(query, page, &user_agent).await?; + // Add a random delay before making the request. + let mut rng = rand::thread_rng(); + let delay_secs = rng.gen_range(1, 10); + std::thread::sleep(Duration::from_secs(delay_secs)); + + // fetch results from upstream search engines simultaneously/concurrently. + let (ddg_map_results, searx_map_results) = join!( + duckduckgo::results(query, page, &user_agent), + searx::results(query, page, &user_agent) + ); + + let ddg_map_results: HashMap = ddg_map_results?; + let searx_map_results: HashMap = searx_map_results?; result_map.extend(ddg_map_results);