✨ feat: rewrite code by implementing common engine trait SearchEngine

src/engines/duckduckgo.rs

@@ -9,7 +9,7 @@ use scraper::{Html, Selector};
 
 use crate::search_results_handler::aggregation_models::RawSearchResult;
 
-use super::engine_models::EngineError;
+use super::engine_models::{EngineError, SearchEngine};
 
 use error_stack::{IntoReport, Report, Result, ResultExt};
 
@@ -30,10 +30,16 @@ use error_stack::{IntoReport, Report, Result, ResultExt};
 /// reach the above `upstream search engine` page or if the `upstream search engine` is unable to
 /// provide results for the requested search query and also returns error if the scraping selector
 /// or HeaderMap fails to initialize.
-pub async fn results(
+
-    query: &str,
+pub struct DuckDuckGo;
+
+#[async_trait::async_trait]
+impl SearchEngine for DuckDuckGo {
+    async fn results(
+        &self,
+        query: String,
         page: u32,
-    user_agent: &str,
+        user_agent: String,
     ) -> Result<HashMap<String, RawSearchResult>, EngineError> {
         // Page number can be missing or empty string and so appropriate handling is required
         // so that upstream server recieves valid page number.
@@ -82,21 +88,9 @@ pub async fn results(
                 .change_context(EngineError::UnexpectedError)?,
         );
 
-    // fetch the html from upstream duckduckgo engine
+        let document: Html = Html::parse_document(
-    let results: String = reqwest::Client::new()
+            &DuckDuckGo::fetch_html_from_upstream(&self, url, header_map).await?,
-        .get(url)
+        );
-        .timeout(Duration::from_secs(5))
-        .headers(header_map) // add spoofed headers to emulate human behaviour
-        .send()
-        .await
-        .into_report()
-        .change_context(EngineError::RequestError)?
-        .text()
-        .await
-        .into_report()
-        .change_context(EngineError::RequestError)?;
-
-    let document: Html = Html::parse_document(&results);
 
         let no_result: Selector = Selector::parse(".no-results")
             .map_err(|_| Report::new(EngineError::UnexpectedError))
@@ -153,3 +147,4 @@ pub async fn results(
             .map(|search_result| (search_result.visiting_url.clone(), search_result))
             .collect())
     }
+}

src/engines/searx.rs

@@ -8,7 +8,7 @@ use std::collections::HashMap;
 
 use crate::search_results_handler::aggregation_models::RawSearchResult;
 
-use super::engine_models::EngineError;
+use super::engine_models::{EngineError, SearchEngine};
 use error_stack::{IntoReport, Report, Result, ResultExt};
 
 /// This function scrapes results from the upstream engine duckduckgo and puts all the scraped
@@ -28,10 +28,16 @@ use error_stack::{IntoReport, Report, Result, ResultExt};
 /// reach the above `upstream search engine` page or if the `upstream search engine` is unable to
 /// provide results for the requested search query and also returns error if the scraping selector
 /// or HeaderMap fails to initialize.
-pub async fn results(
+
-    query: &str,
+pub struct Searx;
+
+#[async_trait::async_trait]
+impl SearchEngine for Searx {
+    async fn results(
+        &self,
+        query: String,
         page: u32,
-    user_agent: &str,
+        user_agent: String,
     ) -> Result<HashMap<String, RawSearchResult>, EngineError> {
         // Page number can be missing or empty string and so appropriate handling is required
         // so that upstream server recieves valid page number.
@@ -62,24 +68,14 @@ pub async fn results(
         );
         header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse().into_report().change_context(EngineError::UnexpectedError)?);
 
-    // fetch the html from upstream searx instance engine
+        let document: Html =
-    let results: String = reqwest::Client::new()
+            Html::parse_document(&Searx::fetch_html_from_upstream(&self, url, header_map).await?);
-        .get(url)
-        .headers(header_map) // add spoofed headers to emulate human behaviours.
-        .send()
-        .await
-        .into_report()
-        .change_context(EngineError::RequestError)?
-        .text()
-        .await
-        .into_report()
-        .change_context(EngineError::RequestError)?;
-
-    let document: Html = Html::parse_document(&results);
 
         let no_result: Selector = Selector::parse("#urls>.dialog-error>p")
             .map_err(|_| Report::new(EngineError::UnexpectedError))
-        .attach_printable_lazy(|| format!("invalid CSS selector: {}", "#urls>.dialog-error>p"))?;
+            .attach_printable_lazy(|| {
+                format!("invalid CSS selector: {}", "#urls>.dialog-error>p")
+            })?;
 
         if let Some(no_result_msg) = document.select(&no_result).nth(1) {
             if no_result_msg.inner_html()
@@ -136,3 +132,4 @@ pub async fn results(
             .map(|search_result| (search_result.visiting_url.clone(), search_result))
             .collect())
     }
+}