From 7def2964b7b88d1f59d34210a9a4a1193e70f55b Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 11 Jul 2023 19:37:31 +0300 Subject: [PATCH 01/24] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20new=20config=20?= =?UTF-8?q?file=20option=20to=20choose=20upstream=20search=20engines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- websurfx/config.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/websurfx/config.lua b/websurfx/config.lua index 3daaa91..467b792 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -26,3 +26,6 @@ theme = "simple" -- the theme name which should be used for the website -- ### Caching ### redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. + +-- ### Search Engines ### +upstream_search_engines = { DuckDuckGo = true, Searx = false } From ff79c1fcfbb7b8db66b183497dd36b56f2c917c8 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 11 Jul 2023 19:38:59 +0300 Subject: [PATCH 02/24] =?UTF-8?q?=E2=9C=A8=20feat:=20code=20to=20parse=20a?= =?UTF-8?q?nd=20handle=20the=20new=20config=20option?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config_parser/parser.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index 4c5c1e6..c3e503d 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -3,7 +3,7 @@ use super::parser_models::Style; use rlua::Lua; -use std::{format, fs, path::Path}; +use std::{collections::HashMap, format, fs, path::Path}; // ------- Constants -------- static COMMON_DIRECTORY_NAME: &str = "websurfx"; @@ -27,6 +27,7 @@ pub struct Config { pub aggregator: AggreatorConfig, pub logging: bool, pub debug: bool, + pub upstream_search_engines: Vec, } /// Configuration options for the aggregator. @@ -75,6 +76,11 @@ impl Config { aggregator: aggregator_config, logging: globals.get::<_, bool>("logging")?, debug: globals.get::<_, bool>("debug")?, + upstream_search_engines: globals + .get::<_, HashMap>("upstream_search_engines")? + .into_iter() + .filter_map(|(key, value)| value.then(|| key)) + .collect(), }) }) } From 1bccffce54c510487c36670110654005c05fc715 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 11 Jul 2023 19:40:21 +0300 Subject: [PATCH 03/24] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20async=5Ftrait?= =?UTF-8?q?=20crate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 273 +++++++++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 1 + 2 files changed, 264 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b19dd2..5670ed9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -268,6 +268,18 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" + [[package]] name = "anyhow" version = "1.0.71" @@ -280,6 +292,17 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" +[[package]] +name = "async-trait" +version = "0.1.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +dependencies = [ + "proc-macro2 1.0.60", + "quote 1.0.28", + "syn 2.0.18", +] + [[package]] name = "autocfg" version = "0.1.8" @@ -423,6 +446,12 @@ dependencies = [ "bytes 1.4.0", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.79" @@ -453,6 +482,59 @@ dependencies = [ "envmnt", ] +[[package]] +name = "ciborium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" + +[[package]] +name = "ciborium-ll" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9394150f5b4273a1763355bd1c2ec54cc5a2593f790587bcd6b2c947cfa9211" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a78fbdd3cc2914ddf37ba444114bc7765bbdcb55ec9cbe6fa054f0137400717" +dependencies = [ + "anstyle", + "bitflags", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" + [[package]] name = "cloudabi" version = "0.0.3" @@ -551,17 +633,74 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.16", +] + [[package]] name = "crossbeam-deque" version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", "maybe-uninit", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch 0.9.15", + "crossbeam-utils 0.8.16", +] + [[package]] name = "crossbeam-epoch" version = "0.8.2" @@ -570,10 +709,23 @@ checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ "autocfg 1.1.0", "cfg-if 0.1.10", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "lazy_static", "maybe-uninit", - "memoffset", + "memoffset 0.5.6", + "scopeguard", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg 1.1.0", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.16", + "memoffset 0.9.0", "scopeguard", ] @@ -584,7 +736,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" dependencies = [ "cfg-if 0.1.10", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "maybe-uninit", ] @@ -599,6 +751,15 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1023,6 +1184,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "handlebars" version = "4.3.7" @@ -1325,6 +1492,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.8" @@ -1502,6 +1678,15 @@ dependencies = [ "autocfg 1.1.0", ] +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg 1.1.0", +] + [[package]] name = "mime" version = "0.3.17" @@ -1660,6 +1845,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "openssl" version = "0.10.54" @@ -1963,6 +2154,34 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "plotters" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" + +[[package]] +name = "plotters-svg" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2214,6 +2433,28 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque 0.8.3", + "crossbeam-utils 0.8.16", + "num_cpus", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -2893,6 +3134,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -2973,7 +3224,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures", ] @@ -3015,7 +3266,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures", "lazy_static", "log", @@ -3058,9 +3309,9 @@ version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" dependencies = [ - "crossbeam-deque", + "crossbeam-deque 0.7.4", "crossbeam-queue", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures", "lazy_static", "log", @@ -3075,7 +3326,7 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures", "slab", "tokio-executor", @@ -3385,6 +3636,8 @@ version = "0.13.0" dependencies = [ "actix-files", "actix-web", + "async-trait", + "criterion", "env_logger", "error-stack", "fake-useragent", diff --git a/Cargo.toml b/Cargo.toml index 6ee7916..9c008d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ md5 = {version="*"} rand={version="*"} once_cell = {version="*"} error-stack = {version="0.3.1"} +async-trait = {version="*"} [dev-dependencies] rusty-hook = "^0.11.2" From b72af01e0e43c47881637a9b73c1df8fa92f2b4c Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 11 Jul 2023 19:41:34 +0300 Subject: [PATCH 04/24] =?UTF-8?q?=E2=9C=A8=20feat:=20implement=20common=20?= =?UTF-8?q?engine=20trait?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/engines/engine_models.rs | 37 +++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/engines/engine_models.rs b/src/engines/engine_models.rs index 7a58688..ebb6454 100644 --- a/src/engines/engine_models.rs +++ b/src/engines/engine_models.rs @@ -1,8 +1,9 @@ //! This module provides the error enum to handle different errors associated while requesting data from //! the upstream search engines with the search query provided by the user. -use error_stack::Context; -use std::fmt; +use crate::search_results_handler::aggregation_models::RawSearchResult; +use error_stack::{IntoReport, Result, ResultExt}; +use std::{collections::HashMap, fmt, time::Duration}; /// A custom error type used for handle engine associated errors. /// @@ -40,4 +41,34 @@ impl fmt::Display for EngineError { } } -impl Context for EngineError {} +impl error_stack::Context for EngineError {} + +#[async_trait::async_trait] +pub trait SearchEngine { + async fn fetch_html_from_upstream( + &self, + url: String, + header_map: reqwest::header::HeaderMap, + ) -> Result { + // fetch the html from upstream search engine + Ok(reqwest::Client::new() + .get(url) + .timeout(Duration::from_secs(5)) + .headers(header_map) // add spoofed headers to emulate human behaviour + .send() + .await + .into_report() + .change_context(EngineError::RequestError)? + .text() + .await + .into_report() + .change_context(EngineError::RequestError)?) + } + + async fn results( + &self, + query: String, + page: u32, + user_agent: String, + ) -> Result, EngineError>; +} From f9b9e87a0e82361c3148ee7cb0a0f24b3bcbe254 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 11 Jul 2023 19:42:17 +0300 Subject: [PATCH 05/24] =?UTF-8?q?=E2=9C=A8=20feat:=20rewrite=20code=20by?= =?UTF-8?q?=20implementing=20common=20engine=20trait=20`SearchEngine`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/engines/duckduckgo.rs | 225 +++++++++++++++++++------------------- src/engines/searx.rs | 193 ++++++++++++++++---------------- 2 files changed, 205 insertions(+), 213 deletions(-) diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index 70c3a87..e13ca4c 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -9,7 +9,7 @@ use scraper::{Html, Selector}; use crate::search_results_handler::aggregation_models::RawSearchResult; -use super::engine_models::EngineError; +use super::engine_models::{EngineError, SearchEngine}; use error_stack::{IntoReport, Report, Result, ResultExt}; @@ -30,126 +30,121 @@ use error_stack::{IntoReport, Report, Result, ResultExt}; /// reach the above `upstream search engine` page or if the `upstream search engine` is unable to /// provide results for the requested search query and also returns error if the scraping selector /// or HeaderMap fails to initialize. -pub async fn results( - query: &str, - page: u32, - user_agent: &str, -) -> Result, EngineError> { - // Page number can be missing or empty string and so appropriate handling is required - // so that upstream server recieves valid page number. - let url: String = match page { - 1 => { - format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js") - } - _ => { - format!( - "https://duckduckgo.com/html/?q={}&s={}&dc={}&v=1&o=json&api=/d.js", - query, - (page / 2 + (page % 2)) * 30, - (page / 2 + (page % 2)) * 30 + 1 - ) - } - }; - // initializing HeaderMap and adding appropriate headers. - let mut header_map = HeaderMap::new(); - header_map.insert( - USER_AGENT, - user_agent - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert( - REFERER, - "https://google.com/" - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert( - CONTENT_TYPE, - "application/x-www-form-urlencoded" - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert( - COOKIE, - "kl=wt-wt" - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); +pub struct DuckDuckGo; - // fetch the html from upstream duckduckgo engine - let results: String = reqwest::Client::new() - .get(url) - .timeout(Duration::from_secs(5)) - .headers(header_map) // add spoofed headers to emulate human behaviour - .send() - .await - .into_report() - .change_context(EngineError::RequestError)? - .text() - .await - .into_report() - .change_context(EngineError::RequestError)?; - - let document: Html = Html::parse_document(&results); - - let no_result: Selector = Selector::parse(".no-results") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".no-results"))?; - - if document.select(&no_result).next().is_some() { - return Err(Report::new(EngineError::EmptyResultSet)); - } - - let results: Selector = Selector::parse(".result") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result"))?; - let result_title: Selector = Selector::parse(".result__a") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result__a"))?; - let result_url: Selector = Selector::parse(".result__url") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result__url"))?; - let result_desc: Selector = Selector::parse(".result__snippet") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result__snippet"))?; - - // scrape all the results from the html - Ok(document - .select(&results) - .map(|result| { - RawSearchResult::new( - result - .select(&result_title) - .next() - .unwrap() - .inner_html() - .trim() - .to_string(), +#[async_trait::async_trait] +impl SearchEngine for DuckDuckGo { + async fn results( + &self, + query: String, + page: u32, + user_agent: String, + ) -> Result, EngineError> { + // Page number can be missing or empty string and so appropriate handling is required + // so that upstream server recieves valid page number. + let url: String = match page { + 1 => { + format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js") + } + _ => { format!( - "https://{}", + "https://duckduckgo.com/html/?q={}&s={}&dc={}&v=1&o=json&api=/d.js", + query, + (page / 2 + (page % 2)) * 30, + (page / 2 + (page % 2)) * 30 + 1 + ) + } + }; + + // initializing HeaderMap and adding appropriate headers. + let mut header_map = HeaderMap::new(); + header_map.insert( + USER_AGENT, + user_agent + .parse() + .into_report() + .change_context(EngineError::UnexpectedError)?, + ); + header_map.insert( + REFERER, + "https://google.com/" + .parse() + .into_report() + .change_context(EngineError::UnexpectedError)?, + ); + header_map.insert( + CONTENT_TYPE, + "application/x-www-form-urlencoded" + .parse() + .into_report() + .change_context(EngineError::UnexpectedError)?, + ); + header_map.insert( + COOKIE, + "kl=wt-wt" + .parse() + .into_report() + .change_context(EngineError::UnexpectedError)?, + ); + + let document: Html = Html::parse_document( + &DuckDuckGo::fetch_html_from_upstream(&self, url, header_map).await?, + ); + + let no_result: Selector = Selector::parse(".no-results") + .map_err(|_| Report::new(EngineError::UnexpectedError)) + .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".no-results"))?; + + if document.select(&no_result).next().is_some() { + return Err(Report::new(EngineError::EmptyResultSet)); + } + + let results: Selector = Selector::parse(".result") + .map_err(|_| Report::new(EngineError::UnexpectedError)) + .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result"))?; + let result_title: Selector = Selector::parse(".result__a") + .map_err(|_| Report::new(EngineError::UnexpectedError)) + .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result__a"))?; + let result_url: Selector = Selector::parse(".result__url") + .map_err(|_| Report::new(EngineError::UnexpectedError)) + .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result__url"))?; + let result_desc: Selector = Selector::parse(".result__snippet") + .map_err(|_| Report::new(EngineError::UnexpectedError)) + .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result__snippet"))?; + + // scrape all the results from the html + Ok(document + .select(&results) + .map(|result| { + RawSearchResult::new( result - .select(&result_url) + .select(&result_title) .next() .unwrap() .inner_html() .trim() - ), - result - .select(&result_desc) - .next() - .unwrap() - .inner_html() - .trim() - .to_string(), - vec!["duckduckgo".to_string()], - ) - }) - .map(|search_result| (search_result.visiting_url.clone(), search_result)) - .collect()) + .to_string(), + format!( + "https://{}", + result + .select(&result_url) + .next() + .unwrap() + .inner_html() + .trim() + ), + result + .select(&result_desc) + .next() + .unwrap() + .inner_html() + .trim() + .to_string(), + vec!["duckduckgo".to_string()], + ) + }) + .map(|search_result| (search_result.visiting_url.clone(), search_result)) + .collect()) + } } diff --git a/src/engines/searx.rs b/src/engines/searx.rs index bc68608..56e3b23 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use crate::search_results_handler::aggregation_models::RawSearchResult; -use super::engine_models::EngineError; +use super::engine_models::{EngineError, SearchEngine}; use error_stack::{IntoReport, Report, Result, ResultExt}; /// This function scrapes results from the upstream engine duckduckgo and puts all the scraped @@ -28,111 +28,108 @@ use error_stack::{IntoReport, Report, Result, ResultExt}; /// reach the above `upstream search engine` page or if the `upstream search engine` is unable to /// provide results for the requested search query and also returns error if the scraping selector /// or HeaderMap fails to initialize. -pub async fn results( - query: &str, - page: u32, - user_agent: &str, -) -> Result, EngineError> { - // Page number can be missing or empty string and so appropriate handling is required - // so that upstream server recieves valid page number. - let url: String = format!("https://searx.work/search?q={query}&pageno={page}"); - // initializing headers and adding appropriate headers. - let mut header_map = HeaderMap::new(); - header_map.insert( - USER_AGENT, - user_agent - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert( - REFERER, - "https://google.com/" - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert( - CONTENT_TYPE, - "application/x-www-form-urlencoded" - .parse() - .into_report() - .change_context(EngineError::UnexpectedError)?, - ); - header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse().into_report().change_context(EngineError::UnexpectedError)?); +pub struct Searx; - // fetch the html from upstream searx instance engine - let results: String = reqwest::Client::new() - .get(url) - .headers(header_map) // add spoofed headers to emulate human behaviours. - .send() - .await - .into_report() - .change_context(EngineError::RequestError)? - .text() - .await - .into_report() - .change_context(EngineError::RequestError)?; +#[async_trait::async_trait] +impl SearchEngine for Searx { + async fn results( + &self, + query: String, + page: u32, + user_agent: String, + ) -> Result, EngineError> { + // Page number can be missing or empty string and so appropriate handling is required + // so that upstream server recieves valid page number. + let url: String = format!("https://searx.work/search?q={query}&pageno={page}"); - let document: Html = Html::parse_document(&results); + // initializing headers and adding appropriate headers. + let mut header_map = HeaderMap::new(); + header_map.insert( + USER_AGENT, + user_agent + .parse() + .into_report() + .change_context(EngineError::UnexpectedError)?, + ); + header_map.insert( + REFERER, + "https://google.com/" + .parse() + .into_report() + .change_context(EngineError::UnexpectedError)?, + ); + header_map.insert( + CONTENT_TYPE, + "application/x-www-form-urlencoded" + .parse() + .into_report() + .change_context(EngineError::UnexpectedError)?, + ); + header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse().into_report().change_context(EngineError::UnexpectedError)?); - let no_result: Selector = Selector::parse("#urls>.dialog-error>p") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", "#urls>.dialog-error>p"))?; + let document: Html = + Html::parse_document(&Searx::fetch_html_from_upstream(&self, url, header_map).await?); - if let Some(no_result_msg) = document.select(&no_result).nth(1) { - if no_result_msg.inner_html() + let no_result: Selector = Selector::parse("#urls>.dialog-error>p") + .map_err(|_| Report::new(EngineError::UnexpectedError)) + .attach_printable_lazy(|| { + format!("invalid CSS selector: {}", "#urls>.dialog-error>p") + })?; + + if let Some(no_result_msg) = document.select(&no_result).nth(1) { + if no_result_msg.inner_html() == "we didn't find any results. Please use another query or search in more categories" { return Err(Report::new(EngineError::EmptyResultSet)); } + } + + let results: Selector = Selector::parse(".result") + .map_err(|_| Report::new(EngineError::UnexpectedError)) + .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result"))?; + let result_title: Selector = Selector::parse("h3>a") + .map_err(|_| Report::new(EngineError::UnexpectedError)) + .attach_printable_lazy(|| format!("invalid CSS selector: {}", "h3>a"))?; + let result_url: Selector = Selector::parse("h3>a") + .map_err(|_| Report::new(EngineError::UnexpectedError)) + .attach_printable_lazy(|| format!("invalid CSS selector: {}", "h3>a"))?; + + let result_desc: Selector = Selector::parse(".content") + .map_err(|_| Report::new(EngineError::UnexpectedError)) + .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".content"))?; + + // scrape all the results from the html + Ok(document + .select(&results) + .map(|result| { + RawSearchResult::new( + result + .select(&result_title) + .next() + .unwrap() + .inner_html() + .trim() + .to_string(), + result + .select(&result_url) + .next() + .unwrap() + .value() + .attr("href") + .unwrap() + .to_string(), + result + .select(&result_desc) + .next() + .unwrap() + .inner_html() + .trim() + .to_string(), + vec!["searx".to_string()], + ) + }) + .map(|search_result| (search_result.visiting_url.clone(), search_result)) + .collect()) } - - let results: Selector = Selector::parse(".result") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result"))?; - let result_title: Selector = Selector::parse("h3>a") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", "h3>a"))?; - let result_url: Selector = Selector::parse("h3>a") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", "h3>a"))?; - - let result_desc: Selector = Selector::parse(".content") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".content"))?; - - // scrape all the results from the html - Ok(document - .select(&results) - .map(|result| { - RawSearchResult::new( - result - .select(&result_title) - .next() - .unwrap() - .inner_html() - .trim() - .to_string(), - result - .select(&result_url) - .next() - .unwrap() - .value() - .attr("href") - .unwrap() - .to_string(), - result - .select(&result_desc) - .next() - .unwrap() - .inner_html() - .trim() - .to_string(), - vec!["searx".to_string()], - ) - }) - .map(|search_result| (search_result.visiting_url.clone(), search_result)) - .collect()) } From 897ab0807f7beac5c1955ed845dab044b1926e8c Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 11 Jul 2023 19:43:32 +0300 Subject: [PATCH 06/24] =?UTF-8?q?=E2=9C=A8=20feat:=20fix=20the=20argument?= =?UTF-8?q?=20type=20used?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/server/routes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/routes.rs b/src/server/routes.rs index 9234d8d..ae511b6 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -128,7 +128,7 @@ pub async fn search( } Err(_) => { let mut results_json: crate::search_results_handler::aggregation_models::SearchResults = - aggregate(query, page, config.aggregator.random_delay, config.debug).await?; + aggregate(query.clone(), page, config.aggregator.random_delay, config.debug, config.upstream_search_engines.clone()).await?; results_json.add_style(config.style.clone()); redis_cache .cache_results(serde_json::to_string(&results_json)?, &page_url)?; From 0781385393e7215352a4407a0f8426cfabb6c075 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 11 Jul 2023 19:44:38 +0300 Subject: [PATCH 07/24] =?UTF-8?q?=E2=9C=A8=20feat:=20implement=20async=20m?= =?UTF-8?q?ultithreading=20and=20engine=20selection=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/search_results_handler/aggregator.rs | 123 ++++++++++++++++------- 1 file changed, 87 insertions(+), 36 deletions(-) diff --git a/src/search_results_handler/aggregator.rs b/src/search_results_handler/aggregator.rs index cba266c..18cd143 100644 --- a/src/search_results_handler/aggregator.rs +++ b/src/search_results_handler/aggregator.rs @@ -3,15 +3,20 @@ use std::{collections::HashMap, time::Duration}; +use error_stack::Report; use rand::Rng; -use tokio::join; +use tokio::task::JoinHandle; use super::{ aggregation_models::{RawSearchResult, SearchResult, SearchResults}, user_agent::random_user_agent, }; -use crate::engines::{duckduckgo, searx}; +use crate::engines::{ + duckduckgo, + engine_models::{EngineError, SearchEngine}, + searx, +}; /// A function that aggregates all the scraped results from the above upstream engines and /// then removes duplicate results and if two results are found to be from two or more engines @@ -37,10 +42,11 @@ use crate::engines::{duckduckgo, searx}; /// function in either `searx` or `duckduckgo` or both otherwise returns a `SearchResults struct` /// containing appropriate values. pub async fn aggregate( - query: &str, + query: String, page: u32, random_delay: bool, debug: bool, + upstream_search_engines: Vec, ) -> Result> { let user_agent: String = random_user_agent(); let mut result_map: HashMap = HashMap::new(); @@ -53,41 +59,86 @@ pub async fn aggregate( } // fetch results from upstream search engines simultaneously/concurrently. - let (ddg_map_results, searx_map_results) = join!( - duckduckgo::results(query, page, &user_agent), - searx::results(query, page, &user_agent) - ); + let search_engines: Vec> = upstream_search_engines + .iter() + .map(|engine| match engine.to_lowercase().as_str() { + "duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box, + "searx " => Box::new(searx::Searx) as Box, + }) + .collect(); - let ddg_map_results = ddg_map_results.unwrap_or_else(|e| { - if debug { - log::error!("Error fetching results from DuckDuckGo: {:?}", e); - } - HashMap::new() - }); - - let searx_map_results = searx_map_results.unwrap_or_else(|e| { - if debug { - log::error!("Error fetching results from Searx: {:?}", e); - } - HashMap::new() - }); - - result_map.extend(ddg_map_results); - - searx_map_results.into_iter().for_each(|(key, value)| { - result_map - .entry(key) - .and_modify(|result| { - result.add_engines(value.clone().engine()); + let tasks: Vec, Report>>> = + search_engines + .iter() + .map(|search_engine| { + tokio::spawn(search_engine.results(query.clone(), page, user_agent.clone())) }) - .or_insert_with(|| -> RawSearchResult { - RawSearchResult::new( - value.title.clone(), - value.visiting_url.clone(), - value.description.clone(), - value.engine.clone(), - ) - }); + .collect(); + + let mut outputs = Vec::with_capacity(search_engines.len()); + + for task in tasks { + outputs.push(task.await.ok()) + } + + let mut initial: bool = true; + let mut counter: usize = 0; + outputs.iter().for_each(|results| { + if initial { + match results { + Some(result) => { + let new_result = result.clone(); + result_map.extend(new_result.as_ref().unwrap().clone()); + counter += 1; + initial = false + } + None => { + if debug { + log::error!( + "Error fetching results from {}", + upstream_search_engines[counter] + ); + }; + counter += 1 + } + } + } else { + match results { + Some(result) => { + let new_result = result.clone(); + new_result + .as_ref() + .unwrap() + .clone() + .into_iter() + .for_each(|(key, value)| { + result_map + .entry(key) + .and_modify(|result| { + result.add_engines(value.clone().engine()); + }) + .or_insert_with(|| -> RawSearchResult { + RawSearchResult::new( + value.title.clone(), + value.visiting_url.clone(), + value.description.clone(), + value.engine.clone(), + ) + }); + }); + counter += 1 + } + None => { + if debug { + log::error!( + "Error fetching results from {}", + upstream_search_engines[counter] + ); + }; + counter += 1 + } + } + } }); Ok(SearchResults::new( From 2f01651be08e8087ab5fab026f1b7d22c0d260fa Mon Sep 17 00:00:00 2001 From: neon_arch Date: Fri, 14 Jul 2023 12:56:06 +0300 Subject: [PATCH 08/24] =?UTF-8?q?=E2=9C=A8=20feat:=20fix=20bugs=20and=20ad?= =?UTF-8?q?d=20code=20to=20handle=20engine=20selections?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sabrina Jewson <58880148+SabrinaJewson@users.noreply.github.com> --- src/search_results_handler/aggregator.rs | 62 ++++++++++++------------ 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/src/search_results_handler/aggregator.rs b/src/search_results_handler/aggregator.rs index 18cd143..aed48aa 100644 --- a/src/search_results_handler/aggregator.rs +++ b/src/search_results_handler/aggregator.rs @@ -59,26 +59,35 @@ pub async fn aggregate( } // fetch results from upstream search engines simultaneously/concurrently. - let search_engines: Vec> = upstream_search_engines + let search_engines: Vec> = upstream_search_engines .iter() .map(|engine| match engine.to_lowercase().as_str() { - "duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box, - "searx " => Box::new(searx::Searx) as Box, + "duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box, + "searx" => Box::new(searx::Searx) as Box, + &_ => panic!("Config Error: Incorrect config file option provided"), }) .collect(); + let task_capacity: usize = search_engines.len(); + let tasks: Vec, Report>>> = search_engines - .iter() + .into_iter() .map(|search_engine| { - tokio::spawn(search_engine.results(query.clone(), page, user_agent.clone())) + let query: String = query.clone(); + let user_agent: String = user_agent.clone(); + tokio::spawn( + async move { search_engine.results(query, page, user_agent.clone()).await }, + ) }) .collect(); - let mut outputs = Vec::with_capacity(search_engines.len()); + let mut outputs = Vec::with_capacity(task_capacity); for task in tasks { - outputs.push(task.await.ok()) + if let Ok(result) = task.await { + outputs.push(result.ok()) + } } let mut initial: bool = true; @@ -87,8 +96,7 @@ pub async fn aggregate( if initial { match results { Some(result) => { - let new_result = result.clone(); - result_map.extend(new_result.as_ref().unwrap().clone()); + result_map.extend(result.clone()); counter += 1; initial = false } @@ -105,27 +113,21 @@ pub async fn aggregate( } else { match results { Some(result) => { - let new_result = result.clone(); - new_result - .as_ref() - .unwrap() - .clone() - .into_iter() - .for_each(|(key, value)| { - result_map - .entry(key) - .and_modify(|result| { - result.add_engines(value.clone().engine()); - }) - .or_insert_with(|| -> RawSearchResult { - RawSearchResult::new( - value.title.clone(), - value.visiting_url.clone(), - value.description.clone(), - value.engine.clone(), - ) - }); - }); + result.clone().into_iter().for_each(|(key, value)| { + result_map + .entry(key) + .and_modify(|result| { + result.add_engines(value.clone().engine()); + }) + .or_insert_with(|| -> RawSearchResult { + RawSearchResult::new( + value.title.clone(), + value.visiting_url.clone(), + value.description.clone(), + value.engine.clone(), + ) + }); + }); counter += 1 } None => { From 23abe619d7dbbd63ef9a46c5054c45d8d95735ba Mon Sep 17 00:00:00 2001 From: neon_arch Date: Fri, 14 Jul 2023 17:14:41 +0300 Subject: [PATCH 09/24] =?UTF-8?q?=E2=9C=A8=20feat:=20enable=20cookies=20fe?= =?UTF-8?q?ature=20for=20actix-web?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9c008d2..0615637 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ tokio = {version="*",features=["full"]} serde = {version="*",features=["derive"]} handlebars = { version = "4.3.6", features = ["dir_source"] } scraper = {version="*"} -actix-web = {version="4.3.1"} +actix-web = {version="4.3.1", features = ["cookies"]} actix-files = {version="0.6.2"} serde_json = {version="*"} fake-useragent = {version="*"} From c223fed39457eed8a10cc00538814155546bc572 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Fri, 14 Jul 2023 17:16:13 +0300 Subject: [PATCH 10/24] =?UTF-8?q?=E2=9C=A8=20feat:=20remove=20the=20time?= =?UTF-8?q?=20crate=20import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/engines/duckduckgo.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index e13ca4c..bcc4b80 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -2,7 +2,7 @@ //! by querying the upstream duckduckgo search engine with user provided query and with a page //! number if provided. -use std::{collections::HashMap, time::Duration}; +use std::collections::HashMap; use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT}; use scraper::{Html, Selector}; From 27247baf0c447ee608c7e39915dd02508cf8f3ff Mon Sep 17 00:00:00 2001 From: neon_arch Date: Fri, 14 Jul 2023 17:17:05 +0300 Subject: [PATCH 11/24] =?UTF-8?q?=E2=9C=A8=20feat:=20define=20common=20beh?= =?UTF-8?q?aviour=20for=20all=20engines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/engines/engine_models.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engines/engine_models.rs b/src/engines/engine_models.rs index ebb6454..bf4b6be 100644 --- a/src/engines/engine_models.rs +++ b/src/engines/engine_models.rs @@ -53,7 +53,7 @@ pub trait SearchEngine { // fetch the html from upstream search engine Ok(reqwest::Client::new() .get(url) - .timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(30)) .headers(header_map) // add spoofed headers to emulate human behaviour .send() .await From 3e6f7867ebd32516fae04851e9bfc5dd319f39f0 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Fri, 14 Jul 2023 17:20:46 +0300 Subject: [PATCH 12/24] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20code=20to=20par?= =?UTF-8?q?se=20cookies=20and=20handle=20engine=20selection=20from=20UI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/server/routes.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/server/routes.rs b/src/server/routes.rs index ae511b6..545e1d0 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -51,6 +51,14 @@ pub async fn not_found( .body(page_content)) } +#[allow(dead_code)] +#[derive(Deserialize)] +struct Cookie { + theme: String, + colorscheme: String, + engines: Vec, +} + /// Handles the route of search page of the `websurfx` meta search engine website and it takes /// two search url parameters `q` and `page` where `page` parameter is optional. /// @@ -127,8 +135,13 @@ pub async fn search( Ok(HttpResponse::Ok().body(page_content)) } Err(_) => { - let mut results_json: crate::search_results_handler::aggregation_models::SearchResults = - aggregate(query.clone(), page, config.aggregator.random_delay, config.debug, config.upstream_search_engines.clone()).await?; + let mut results_json: crate::search_results_handler::aggregation_models::SearchResults = match req.cookie("appCookie") { + Some(cookie_value) => { + let cookie_value:Cookie = serde_json::from_str(cookie_value.name_value().1).unwrap(); + aggregate(query.clone(), page, config.aggregator.random_delay, config.debug, cookie_value.engines).await? + }, + None => aggregate(query.clone(), page, config.aggregator.random_delay, config.debug, config.upstream_search_engines.clone()).await?, + }; results_json.add_style(config.style.clone()); redis_cache .cache_results(serde_json::to_string(&results_json)?, &page_url)?; From 9ee516e9b1359a35052bdadc549433d35d2c796c Mon Sep 17 00:00:00 2001 From: neon_arch Date: Fri, 14 Jul 2023 21:18:26 +0300 Subject: [PATCH 13/24] =?UTF-8?q?=E2=9C=A8=20feat:=20fix=20code=20for=20cl?= =?UTF-8?q?ippy=20suggestions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config_parser/parser.rs | 2 +- src/engines/duckduckgo.rs | 2 +- src/engines/searx.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index c3e503d..fd94673 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -79,7 +79,7 @@ impl Config { upstream_search_engines: globals .get::<_, HashMap>("upstream_search_engines")? .into_iter() - .filter_map(|(key, value)| value.then(|| key)) + .filter_map(|(key, value)| value.then_some(key)) .collect(), }) }) diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index bcc4b80..61ff9cb 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -89,7 +89,7 @@ impl SearchEngine for DuckDuckGo { ); let document: Html = Html::parse_document( - &DuckDuckGo::fetch_html_from_upstream(&self, url, header_map).await?, + &DuckDuckGo::fetch_html_from_upstream(self, url, header_map).await?, ); let no_result: Selector = Selector::parse(".no-results") diff --git a/src/engines/searx.rs b/src/engines/searx.rs index 56e3b23..71bc80d 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -69,7 +69,7 @@ impl SearchEngine for Searx { header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse().into_report().change_context(EngineError::UnexpectedError)?); let document: Html = - Html::parse_document(&Searx::fetch_html_from_upstream(&self, url, header_map).await?); + Html::parse_document(&Searx::fetch_html_from_upstream(self, url, header_map).await?); let no_result: Selector = Selector::parse("#urls>.dialog-error>p") .map_err(|_| Report::new(EngineError::UnexpectedError)) From 3aeb3b3e732481e09202cf500adfd63e374bd44b Mon Sep 17 00:00:00 2001 From: neon_arch Date: Fri, 14 Jul 2023 21:26:29 +0300 Subject: [PATCH 14/24] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20setter=20functi?= =?UTF-8?q?ons=20to=20SearchResults=20struct?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../aggregation_models.rs | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/src/search_results_handler/aggregation_models.rs b/src/search_results_handler/aggregation_models.rs index b6e6b81..31a43c2 100644 --- a/src/search_results_handler/aggregation_models.rs +++ b/src/search_results_handler/aggregation_models.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; -use crate::config_parser::parser_models::Style; +use crate::{config_parser::parser_models::Style, engines::engine_models::EngineError}; /// A named struct to store, serialize and deserializes the individual search result from all the /// scraped and aggregated search results from the upstream search engines. @@ -16,7 +16,7 @@ use crate::config_parser::parser_models::Style; /// * `url` - The url to be displayed below the search result title in html. /// * `description` - The description of the search result. /// * `engine` - The names of the upstream engines from which this results were provided. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SearchResult { pub title: String, @@ -116,6 +116,25 @@ impl RawSearchResult { } } +#[derive(Serialize, Deserialize)] +pub struct EngineErrorInfo { + pub error: String, + pub engine: String, +} + +impl EngineErrorInfo { + pub fn new(error: &EngineError, engine: String) -> Self { + Self { + error: match error { + EngineError::RequestError => String::from("RequestError"), + EngineError::EmptyResultSet => String::from("EmptyResultSet"), + EngineError::UnexpectedError => String::from("UnexpectedError"), + }, + engine, + } + } +} + /// A named struct to store, serialize, deserialize the all the search results scraped and /// aggregated from the upstream search engines. /// @@ -130,6 +149,8 @@ pub struct SearchResults { pub results: Vec, pub page_query: String, pub style: Style, + pub engine_errors_info: Vec, + pub empty_result_set: bool, } impl SearchResults { @@ -141,15 +162,29 @@ impl SearchResults { /// and stores it into a vector of `SearchResult` structs. /// * `page_query` - Takes an argument of current page`s search query `q` provided in /// the search url. - pub fn new(results: Vec, page_query: String) -> Self { + pub fn new( + results: Vec, + page_query: String, + engine_errors_info: Vec, + ) -> Self { SearchResults { results, page_query, style: Style::new("".to_string(), "".to_string()), + engine_errors_info, + empty_result_set: false, } } pub fn add_style(&mut self, style: Style) { self.style = style; } + + pub fn is_empty_result_set(&self) -> bool { + self.results.is_empty() + } + + pub fn set_empty_result_set(&mut self) { + self.empty_result_set = true; + } } From a3edf70d1efb264bca7cab64d0fe55158edfa089 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Fri, 14 Jul 2023 21:27:23 +0300 Subject: [PATCH 15/24] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20code=20to=20pas?= =?UTF-8?q?s=20engine=20errors=20to=20the=20UI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/search_results_handler/aggregator.rs | 58 ++++++++++++------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/search_results_handler/aggregator.rs b/src/search_results_handler/aggregator.rs index aed48aa..197c3d7 100644 --- a/src/search_results_handler/aggregator.rs +++ b/src/search_results_handler/aggregator.rs @@ -8,7 +8,7 @@ use rand::Rng; use tokio::task::JoinHandle; use super::{ - aggregation_models::{RawSearchResult, SearchResult, SearchResults}, + aggregation_models::{EngineErrorInfo, RawSearchResult, SearchResult, SearchResults}, user_agent::random_user_agent, }; @@ -18,6 +18,8 @@ use crate::engines::{ searx, }; +type FutureVec = Vec, Report>>>; + /// A function that aggregates all the scraped results from the above upstream engines and /// then removes duplicate results and if two results are found to be from two or more engines /// then puts their names together to show the results are fetched from these upstream engines @@ -70,49 +72,48 @@ pub async fn aggregate( let task_capacity: usize = search_engines.len(); - let tasks: Vec, Report>>> = - search_engines - .into_iter() - .map(|search_engine| { - let query: String = query.clone(); - let user_agent: String = user_agent.clone(); - tokio::spawn( - async move { search_engine.results(query, page, user_agent.clone()).await }, - ) - }) - .collect(); + let tasks: FutureVec = search_engines + .into_iter() + .map(|search_engine| { + let query: String = query.clone(); + let user_agent: String = user_agent.clone(); + tokio::spawn( + async move { search_engine.results(query, page, user_agent.clone()).await }, + ) + }) + .collect(); let mut outputs = Vec::with_capacity(task_capacity); for task in tasks { if let Ok(result) = task.await { - outputs.push(result.ok()) + outputs.push(result) } } + let mut engine_errors_info: Vec = Vec::new(); + let mut initial: bool = true; let mut counter: usize = 0; outputs.iter().for_each(|results| { if initial { match results { - Some(result) => { + Ok(result) => { result_map.extend(result.clone()); counter += 1; initial = false } - None => { - if debug { - log::error!( - "Error fetching results from {}", - upstream_search_engines[counter] - ); - }; + Err(error_type) => { + engine_errors_info.push(EngineErrorInfo::new( + error_type.downcast_ref::().unwrap(), + upstream_search_engines[counter].clone(), + )); counter += 1 } } } else { match results { - Some(result) => { + Ok(result) => { result.clone().into_iter().for_each(|(key, value)| { result_map .entry(key) @@ -130,13 +131,11 @@ pub async fn aggregate( }); counter += 1 } - None => { - if debug { - log::error!( - "Error fetching results from {}", - upstream_search_engines[counter] - ); - }; + Err(error_type) => { + engine_errors_info.push(EngineErrorInfo::new( + error_type.downcast_ref::().unwrap(), + upstream_search_engines[counter].clone(), + )); counter += 1 } } @@ -157,5 +156,6 @@ pub async fn aggregate( }) .collect(), query.to_string(), + engine_errors_info, )) } From 8133de1758af72c2d60d2be46ca3269f5cd79587 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Fri, 14 Jul 2023 21:29:01 +0300 Subject: [PATCH 16/24] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20check=20to=20co?= =?UTF-8?q?nditionaly=20pass=20empty=5Fresult=5Fset=20to=20true=20to=20UI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/server/routes.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/server/routes.rs b/src/server/routes.rs index 545e1d0..c614139 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -22,7 +22,7 @@ use serde::Deserialize; /// of the search url. /// * `page` - It stores the search parameter `page` (or pageno in simple words) /// of the search url. -#[derive(Debug, Deserialize)] +#[derive(Deserialize)] struct SearchParams { q: Option, page: Option, @@ -137,12 +137,15 @@ pub async fn search( Err(_) => { let mut results_json: crate::search_results_handler::aggregation_models::SearchResults = match req.cookie("appCookie") { Some(cookie_value) => { - let cookie_value:Cookie = serde_json::from_str(cookie_value.name_value().1).unwrap(); - aggregate(query.clone(), page, config.aggregator.random_delay, config.debug, cookie_value.engines).await? + let cookie_value:Cookie = serde_json::from_str(cookie_value.name_value().1)?; + aggregate(query.clone(), page, config.aggregator.random_delay, config.debug, cookie_value.engines).await? }, None => aggregate(query.clone(), page, config.aggregator.random_delay, config.debug, config.upstream_search_engines.clone()).await?, }; results_json.add_style(config.style.clone()); + if results_json.is_empty_result_set() { + results_json.set_empty_result_set(); + } redis_cache .cache_results(serde_json::to_string(&results_json)?, &page_url)?; let page_content: String = hbs.render("search", &results_json)?; From 94ef62eec918cdbba9e45d189ade83241d499fe5 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sat, 15 Jul 2023 13:36:46 +0300 Subject: [PATCH 17/24] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20documentation?= =?UTF-8?q?=20to=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config_parser/parser.rs | 10 ++++- src/engines/duckduckgo.rs | 37 +++++++++--------- src/engines/engine_models.rs | 3 +- src/engines/searx.rs | 38 ++++++++++--------- .../aggregation_models.rs | 10 +++++ src/search_results_handler/aggregator.rs | 22 ++++++++--- src/server/routes.rs | 16 +++++++- 7 files changed, 91 insertions(+), 45 deletions(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index fd94673..9bb0150 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -18,6 +18,10 @@ static CONFIG_FILE_NAME: &str = "config.lua"; /// * `style` - It stores the theming options for the website. /// * `redis_connection_url` - It stores the redis connection url address on which the redis /// client should connect. +/// * `aggregator` - It stores the option to whether enable or disable production use. +/// * `logging` - It stores the option to whether enable or disable logs. +/// * `debug` - It stores the option to whether enable or disable debug mode. +/// * `upstream_search_engines` - It stores all the engine names that were enabled by the user. #[derive(Clone)] pub struct Config { pub port: u16, @@ -31,9 +35,13 @@ pub struct Config { } /// Configuration options for the aggregator. +/// +/// # Fields +/// +/// * `random_delay` - It stores the option to whether enable or disable random delays between +/// requests. #[derive(Clone)] pub struct AggreatorConfig { - /// Whether to introduce a random delay before sending the request to the search engine. pub random_delay: bool, } diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index 61ff9cb..bbe561f 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -13,28 +13,29 @@ use super::engine_models::{EngineError, SearchEngine}; use error_stack::{IntoReport, Report, Result, ResultExt}; -/// This function scrapes results from the upstream engine duckduckgo and puts all the scraped -/// results like title, visiting_url (href in html),engine (from which engine it was fetched from) -/// and description in a RawSearchResult and then adds that to HashMap whose keys are url and -/// values are RawSearchResult struct and then returns it within a Result enum. -/// -/// # Arguments -/// -/// * `query` - Takes the user provided query to query to the upstream search engine with. -/// * `page` - Takes an u32 as an argument. -/// * `user_agent` - Takes a random user agent string as an argument. -/// -/// # Errors -/// -/// Returns an `EngineErrorKind` if the user is not connected to the internet or if their is failure to -/// reach the above `upstream search engine` page or if the `upstream search engine` is unable to -/// provide results for the requested search query and also returns error if the scraping selector -/// or HeaderMap fails to initialize. - +/// A new DuckDuckGo engine type defined in-order to implement the `SearchEngine` trait which allows to +/// reduce code duplication as well as allows to create vector of different search engines easily. pub struct DuckDuckGo; #[async_trait::async_trait] impl SearchEngine for DuckDuckGo { + /// This function scrapes results from the upstream engine duckduckgo and puts all the scraped + /// results like title, visiting_url (href in html),engine (from which engine it was fetched from) + /// and description in a RawSearchResult and then adds that to HashMap whose keys are url and + /// values are RawSearchResult struct and then returns it within a Result enum. + /// + /// # Arguments + /// + /// * `query` - Takes the user provided query to query to the upstream search engine with. + /// * `page` - Takes an u32 as an argument. + /// * `user_agent` - Takes a random user agent string as an argument. + /// + /// # Errors + /// + /// Returns an `EngineErrorKind` if the user is not connected to the internet or if their is failure to + /// reach the above `upstream search engine` page or if the `upstream search engine` is unable to + /// provide results for the requested search query and also returns error if the scraping selector + /// or HeaderMap fails to initialize. async fn results( &self, query: String, diff --git a/src/engines/engine_models.rs b/src/engines/engine_models.rs index bf4b6be..e6eb50f 100644 --- a/src/engines/engine_models.rs +++ b/src/engines/engine_models.rs @@ -43,6 +43,7 @@ impl fmt::Display for EngineError { impl error_stack::Context for EngineError {} +/// A trait to define common behaviour for all search engines. #[async_trait::async_trait] pub trait SearchEngine { async fn fetch_html_from_upstream( @@ -53,7 +54,7 @@ pub trait SearchEngine { // fetch the html from upstream search engine Ok(reqwest::Client::new() .get(url) - .timeout(Duration::from_secs(30)) + .timeout(Duration::from_secs(30)) // Add timeout to request to avoid DDOSing the server .headers(header_map) // add spoofed headers to emulate human behaviour .send() .await diff --git a/src/engines/searx.rs b/src/engines/searx.rs index 71bc80d..6706e3c 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -11,28 +11,30 @@ use crate::search_results_handler::aggregation_models::RawSearchResult; use super::engine_models::{EngineError, SearchEngine}; use error_stack::{IntoReport, Report, Result, ResultExt}; -/// This function scrapes results from the upstream engine duckduckgo and puts all the scraped -/// results like title, visiting_url (href in html),engine (from which engine it was fetched from) -/// and description in a RawSearchResult and then adds that to HashMap whose keys are url and -/// values are RawSearchResult struct and then returns it within a Result enum. -/// -/// # Arguments -/// -/// * `query` - Takes the user provided query to query to the upstream search engine with. -/// * `page` - Takes an u32 as an argument. -/// * `user_agent` - Takes a random user agent string as an argument. -/// -/// # Errors -/// -/// Returns an `EngineErrorKind` if the user is not connected to the internet or if their is failure to -/// reach the above `upstream search engine` page or if the `upstream search engine` is unable to -/// provide results for the requested search query and also returns error if the scraping selector -/// or HeaderMap fails to initialize. - +/// A new Searx engine type defined in-order to implement the `SearchEngine` trait which allows to +/// reduce code duplication as well as allows to create vector of different search engines easily. pub struct Searx; #[async_trait::async_trait] impl SearchEngine for Searx { + /// This function scrapes results from the upstream engine duckduckgo and puts all the scraped + /// results like title, visiting_url (href in html),engine (from which engine it was fetched from) + /// and description in a RawSearchResult and then adds that to HashMap whose keys are url and + /// values are RawSearchResult struct and then returns it within a Result enum. + /// + /// # Arguments + /// + /// * `query` - Takes the user provided query to query to the upstream search engine with. + /// * `page` - Takes an u32 as an argument. + /// * `user_agent` - Takes a random user agent string as an argument. + /// + /// # Errors + /// + /// Returns an `EngineErrorKind` if the user is not connected to the internet or if their is failure to + /// reach the above `upstream search engine` page or if the `upstream search engine` is unable to + /// provide results for the requested search query and also returns error if the scraping selector + /// or HeaderMap fails to initialize. + async fn results( &self, query: String, diff --git a/src/search_results_handler/aggregation_models.rs b/src/search_results_handler/aggregation_models.rs index 31a43c2..4177cc3 100644 --- a/src/search_results_handler/aggregation_models.rs +++ b/src/search_results_handler/aggregation_models.rs @@ -143,6 +143,11 @@ impl EngineErrorInfo { /// * `results` - Stores the individual serializable `SearchResult` struct into a vector of /// `SearchResult` structs. /// * `page_query` - Stores the current pages search query `q` provided in the search url. +/// * `style` - Stores the theming options for the website. +/// * `engine_errors_info` - Stores the information on which engines failed with their engine name +/// and the type of error that caused it. +/// * `empty_result_set` - Stores a boolean which indicates that no engines gave a result for the +/// given search query. #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SearchResults { @@ -162,6 +167,8 @@ impl SearchResults { /// and stores it into a vector of `SearchResult` structs. /// * `page_query` - Takes an argument of current page`s search query `q` provided in /// the search url. + /// * `empty_result_set` - Takes a boolean which indicates that no engines gave a result for the + /// given search query. pub fn new( results: Vec, page_query: String, @@ -176,14 +183,17 @@ impl SearchResults { } } + /// A setter function to add website style to the return search results. pub fn add_style(&mut self, style: Style) { self.style = style; } + /// A function which checks whether the results stored are empty or not. pub fn is_empty_result_set(&self) -> bool { self.results.is_empty() } + /// A setter function which sets the empty_result_set to true. pub fn set_empty_result_set(&mut self) { self.empty_result_set = true; } diff --git a/src/search_results_handler/aggregator.rs b/src/search_results_handler/aggregator.rs index 197c3d7..ec5c065 100644 --- a/src/search_results_handler/aggregator.rs +++ b/src/search_results_handler/aggregator.rs @@ -18,14 +18,21 @@ use crate::engines::{ searx, }; +/// Aliases for long type annotations type FutureVec = Vec, Report>>>; -/// A function that aggregates all the scraped results from the above upstream engines and -/// then removes duplicate results and if two results are found to be from two or more engines -/// then puts their names together to show the results are fetched from these upstream engines -/// and then removes all data from the HashMap and puts into a struct of all results aggregated -/// into a vector and also adds the query used into the struct this is neccessory because -/// otherwise the search bar in search remains empty if searched from the query url +/// A function that aggregates all the scraped results from the above user selected upstream +/// search engines either selected from the UI or from the config file which is handled by the code +/// by matching over the selected search engines and adding the selected ones to the vector which +/// is then used to create an async task vector with `tokio::spawn` which returns a future which +/// is then awaited on in another loop and then all the collected results is filtered for errors +/// and proper results and if an error is found is then sent to the UI with the engine name and the +/// error type that caused it by putting them finallt in the returned `SearchResults` struct. Also +/// the same process also removes duplicate results and if two results are found to be from two or +/// more engines then puts their names together to show the results are fetched from these upstream +/// engines and then removes all data from the HashMap and puts into a struct of all results aggregated +/// into a vector and also adds the query used into the struct this is neccessory because otherwise the +/// search bar in search remains empty if searched from the query url. /// /// # Example: /// @@ -37,6 +44,9 @@ type FutureVec = Vec, Report< /// * `query` - Accepts a string to query with the above upstream search engines. /// * `page` - Accepts an u32 page number. /// * `random_delay` - Accepts a boolean value to add a random delay before making the request. +/// * `debug` - Accepts a boolean value to enable or disable debug mode option. +/// * `upstream_search_engines` - Accepts a vector of search engine names which was selected by the +/// user through the UI or the config file. /// /// # Error /// diff --git a/src/server/routes.rs b/src/server/routes.rs index c614139..ada2ef2 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -51,6 +51,13 @@ pub async fn not_found( .body(page_content)) } +/// A named struct which is used to deserialize the cookies fetched from the client side. +/// +/// # Fields +/// +/// * `theme` - It stores the theme name used in the website. +/// * `colorscheme` - It stores the colorscheme name used for the website theme. +/// * `engines` - It stores the user selected upstream search engines selected from the UI. #[allow(dead_code)] #[derive(Deserialize)] struct Cookie { @@ -126,7 +133,7 @@ pub async fn search( // fetch the cached results json. let cached_results_json = redis_cache.cached_results_json(&page_url); - // check if fetched results was indeed fetched or it was an error and if so + // check if fetched catch results was indeed fetched or it was an error and if so // handle the data accordingly. match cached_results_json { Ok(results_json) => { @@ -135,6 +142,10 @@ pub async fn search( Ok(HttpResponse::Ok().body(page_content)) } Err(_) => { + // check if the cookie value is empty or not if it is empty then use the + // default selected upstream search engines from the config file otherwise + // parse the non-empty cookie and grab the user selected engines from the + // UI and use that. let mut results_json: crate::search_results_handler::aggregation_models::SearchResults = match req.cookie("appCookie") { Some(cookie_value) => { let cookie_value:Cookie = serde_json::from_str(cookie_value.name_value().1)?; @@ -143,6 +154,9 @@ pub async fn search( None => aggregate(query.clone(), page, config.aggregator.random_delay, config.debug, config.upstream_search_engines.clone()).await?, }; results_json.add_style(config.style.clone()); + // check whether the results grabbed from the upstream engines are empty or + // not if they are empty then set the empty_result_set option to true in + // the result json. if results_json.is_empty_result_set() { results_json.set_empty_result_set(); } From 0096154bddbb433f5acad90d2e625c40cc825b96 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sat, 15 Jul 2023 13:37:53 +0300 Subject: [PATCH 18/24] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20documentation?= =?UTF-8?q?=20for=20the=20new=20config=20option?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- websurfx/config.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/websurfx/config.lua b/websurfx/config.lua index 467b792..26ab979 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -5,7 +5,7 @@ debug = false -- an option to enable or disable debug mode. -- ### Server ### port = "8080" -- port on which server should be launched binding_ip_addr = "127.0.0.1" --ip address on the which server should be launched. -production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) +production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users (more than one)) -- if production_use is set to true -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. @@ -28,4 +28,4 @@ theme = "simple" -- the theme name which should be used for the website redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. -- ### Search Engines ### -upstream_search_engines = { DuckDuckGo = true, Searx = false } +upstream_search_engines = { DuckDuckGo = true, Searx = false } -- select the upstream search engines from which the results should be fetched. From 1c2ea24024cfe4e1b63763037de0129f09b40de3 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sat, 15 Jul 2023 13:43:49 +0300 Subject: [PATCH 19/24] =?UTF-8?q?=F0=9F=9A=80=20chore:=20bump=20the=20app?= =?UTF-8?q?=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 737 ++++++++++++++++++++++------------------------------- Cargo.toml | 2 +- 2 files changed, 302 insertions(+), 437 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5670ed9..6552adc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,13 +8,13 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bytes 1.4.0", "futures-core", "futures-sink", "memchr", "pin-project-lite", - "tokio 1.28.2", + "tokio 1.29.1", "tokio-util", "tracing", ] @@ -30,7 +30,7 @@ dependencies = [ "actix-utils", "actix-web", "askama_escape", - "bitflags", + "bitflags 1.3.2", "bytes 1.4.0", "derive_more", "futures-core", @@ -54,7 +54,7 @@ dependencies = [ "actix-utils", "ahash 0.8.3", "base64 0.21.2", - "bitflags", + "bitflags 1.3.2", "brotli", "bytes 1.4.0", "bytestring", @@ -62,11 +62,11 @@ dependencies = [ "encoding_rs", "flate2", "futures-core", - "h2 0.3.19", + "h2 0.3.20", "http 0.2.9", "httparse", "httpdate", - "itoa 1.0.6", + "itoa 1.0.8", "language-tags", "local-channel", "mime", @@ -74,8 +74,8 @@ dependencies = [ "pin-project-lite", "rand 0.8.5", "sha1", - "smallvec 1.10.0", - "tokio 1.28.2", + "smallvec 1.11.0", + "tokio 1.29.1", "tokio-util", "tracing", "zstd", @@ -87,7 +87,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" dependencies = [ - "quote 1.0.28", + "quote 1.0.29", "syn 1.0.109", ] @@ -111,7 +111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" dependencies = [ "futures-core", - "tokio 1.28.2", + "tokio 1.29.1", ] [[package]] @@ -128,7 +128,7 @@ dependencies = [ "mio 0.8.8", "num_cpus", "socket2", - "tokio 1.28.2", + "tokio 1.29.1", "tracing", ] @@ -178,7 +178,7 @@ dependencies = [ "futures-core", "futures-util", "http 0.2.9", - "itoa 1.0.6", + "itoa 1.0.8", "language-tags", "log", "mime", @@ -188,9 +188,9 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.1", - "smallvec 1.10.0", + "smallvec 1.11.0", "socket2", - "time 0.3.22", + "time 0.3.23", "url 2.4.0", ] @@ -201,16 +201,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" dependencies = [ "actix-router", - "proc-macro2 1.0.60", - "quote 1.0.28", + "proc-macro2 1.0.64", + "quote 1.0.29", "syn 1.0.109", ] [[package]] name = "addr2line" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" dependencies = [ "gimli", ] @@ -227,7 +227,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.10", + "getrandom", "once_cell", "version_check", ] @@ -239,7 +239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ "cfg-if 1.0.0", - "getrandom 0.2.10", + "getrandom", "once_cell", "version_check", ] @@ -294,13 +294,13 @@ checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" dependencies = [ - "proc-macro2 1.0.60", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.64", + "quote 1.0.29", + "syn 2.0.26", ] [[package]] @@ -320,15 +320,15 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" dependencies = [ "addr2line", "cc", "cfg-if 1.0.0", "libc", - "miniz_oxide 0.6.2", + "miniz_oxide", "object", "rustc-demangle", ] @@ -369,6 +369,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + [[package]] name = "block-buffer" version = "0.10.4" @@ -511,21 +517,20 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.8" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9394150f5b4273a1763355bd1c2ec54cc5a2593f790587bcd6b2c947cfa9211" +checksum = "3eab9e8ceb9afdade1ab3f0fd8dbce5b1b2f468ad653baf10e771781b2b67b73" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.3.8" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a78fbdd3cc2914ddf37ba444114bc7765bbdcb55ec9cbe6fa054f0137400717" +checksum = "9f2763db829349bf00cfc06251268865ed4363b93a943174f638daf3ecdba2cd" dependencies = [ "anstyle", - "bitflags", "clap_lex", ] @@ -541,7 +546,7 @@ version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -577,7 +582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding 2.3.0", - "time 0.3.22", + "time 0.3.23", "version_check", ] @@ -617,9 +622,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -772,19 +777,15 @@ dependencies = [ [[package]] name = "cssparser" -version = "0.29.6" +version = "0.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93d03419cb5950ccfd3daf3ff1c7a36ace64609a1a8746d493df1ca0afde0fa" +checksum = "5b3df4f93e5fbbe73ec01ec8d3f68bba73107993a5b1e7519273c32db9b0d5be" dependencies = [ "cssparser-macros", "dtoa-short", - "itoa 1.0.6", - "matches", - "phf 0.10.1", - "proc-macro2 1.0.60", - "quote 1.0.28", - "smallvec 1.10.0", - "syn 1.0.109", + "itoa 1.0.8", + "phf 0.11.2", + "smallvec 1.11.0", ] [[package]] @@ -793,8 +794,8 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ - "quote 1.0.28", - "syn 2.0.18", + "quote 1.0.29", + "syn 2.0.26", ] [[package]] @@ -804,8 +805,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.60", - "quote 1.0.28", + "proc-macro2 1.0.64", + "quote 1.0.29", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -828,9 +829,9 @@ checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" [[package]] name = "dtoa" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65d09067bfacaa79114679b279d7f5885b53295b1e2cfb4e79c8e4bd3d633169" +checksum = "519b83cd10f5f6e969625a409f735182bea5558cd8b64c655806ceaae36f1999" [[package]] name = "dtoa-short" @@ -838,7 +839,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbaceec3c6e4211c79e7b1800fb9680527106beb2f9c51904a3210c03a448c74" dependencies = [ - "dtoa 1.0.6", + "dtoa 1.0.8", ] [[package]] @@ -893,7 +894,7 @@ checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -932,8 +933,8 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.60", - "quote 1.0.28", + "proc-macro2 1.0.64", + "quote 1.0.29", "syn 1.0.109", "synstructure", ] @@ -965,7 +966,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", - "miniz_oxide 0.7.1", + "miniz_oxide", ] [[package]] @@ -1016,7 +1017,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" dependencies = [ - "bitflags", + "bitflags 1.3.2", "fuchsia-zircon-sys", ] @@ -1119,17 +1120,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.10" @@ -1167,9 +1157,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ "bytes 1.4.0", "fnv", @@ -1179,7 +1169,7 @@ dependencies = [ "http 0.2.9", "indexmap", "slab", - "tokio 1.28.2", + "tokio 1.29.1", "tokio-util", "tracing", ] @@ -1213,18 +1203,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hermit-abi" -version = "0.2.6" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "html5ever" @@ -1249,8 +1230,8 @@ dependencies = [ "log", "mac", "markup5ever 0.11.0", - "proc-macro2 1.0.60", - "quote 1.0.28", + "proc-macro2 1.0.64", + "quote 1.0.29", "syn 1.0.109", ] @@ -1273,7 +1254,7 @@ checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes 1.4.0", "fnv", - "itoa 1.0.6", + "itoa 1.0.8", ] [[package]] @@ -1355,23 +1336,23 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes 1.4.0", "futures-channel", "futures-core", "futures-util", - "h2 0.3.19", + "h2 0.3.20", "http 0.2.9", "http-body 0.4.5", "httparse", "httpdate", - "itoa 1.0.6", + "itoa 1.0.8", "pin-project-lite", "socket2", - "tokio 1.28.2", + "tokio 1.29.1", "tower-service", "tracing", "want 0.3.1", @@ -1397,9 +1378,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.4.0", - "hyper 0.14.26", + "hyper 0.14.27", "native-tls", - "tokio 1.28.2", + "tokio 1.29.1", "tokio-native-tls", ] @@ -1460,9 +1441,9 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.1", + "hermit-abi", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -1476,20 +1457,19 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.7.2" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "is-terminal" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys 0.48.0", + "hermit-abi", + "rustix 0.38.4", + "windows-sys", ] [[package]] @@ -1509,9 +1489,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" [[package]] name = "jobserver" @@ -1555,9 +1535,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "linux-raw-sys" @@ -1565,6 +1545,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +[[package]] +name = "linux-raw-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" + [[package]] name = "local-channel" version = "0.1.3" @@ -1703,15 +1689,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "miniz_oxide" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" -dependencies = [ - "adler", -] - [[package]] name = "miniz_oxide" version = "0.7.1" @@ -1749,7 +1726,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -1805,12 +1782,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab250442c86f1850815b5d268639dff018c0627022bc1940eb2d642ca1ce12f0" -[[package]] -name = "nodrop" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" - [[package]] name = "num-traits" version = "0.2.15" @@ -1822,19 +1793,19 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] [[package]] name = "object" -version = "0.30.4" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" dependencies = [ "memchr", ] @@ -1853,11 +1824,11 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.54" +version = "0.10.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69b3f656a17a6cbc115b5c7a40c616947d213ba182135b014d6051b73ab6f019" +checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if 1.0.0", "foreign-types", "libc", @@ -1872,9 +1843,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.60", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.64", + "quote 1.0.29", + "syn 2.0.26", ] [[package]] @@ -1885,9 +1856,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.88" +version = "0.9.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ce0f250f34a308dcfdbb351f511359857d4ed2134ba715a4eadd46e1ffd617" +checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" dependencies = [ "cc", "libc", @@ -1940,15 +1911,15 @@ dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall 0.3.5", - "smallvec 1.10.0", + "smallvec 1.11.0", "windows-targets", ] [[package]] name = "paste" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "b4b27ab7be369122c218afc2079489cdcb4b517c0a3fc386ff11e1fedfcc2b35" [[package]] name = "percent-encoding" @@ -1964,9 +1935,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16833386b02953ca926d19f64af613b9bf742c48dcd5e09b32fbfc9740bf84e2" +checksum = "f73935e4d55e2abf7f130186537b19e7a4abc886a0252380b59248af473a3fc9" dependencies = [ "thiserror", "ucd-trie", @@ -1974,9 +1945,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7763190f9406839f99e5197afee8c9e759969f7dbfa40ad3b8dbee8757b745b5" +checksum = "aef623c9bbfa0eedf5a0efba11a5ee83209c326653ca31ff019bec3a95bfff2b" dependencies = [ "pest", "pest_generator", @@ -1984,22 +1955,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249061b22e99973da1f5f5f1410284419e283bb60b79255bf5f42a94b66a2e00" +checksum = "b3e8cba4ec22bada7fc55ffe51e2deb6a0e0db2d0b7ab0b103acc80d2510c190" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.60", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.64", + "quote 1.0.29", + "syn 2.0.26", ] [[package]] name = "pest_meta" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "457c310cfc9cf3f22bc58901cc7f0d3410ac5d6298e432a4f9a6138565cb6df6" +checksum = "a01f71cb40bd8bb94232df14b946909e14660e33fc05db3e50ae2a82d7ea0ca0" dependencies = [ "once_cell", "pest", @@ -2015,24 +1986,23 @@ dependencies = [ "phf_shared 0.7.24", ] -[[package]] -name = "phf" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" -dependencies = [ - "phf_shared 0.8.0", -] - [[package]] name = "phf" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" dependencies = [ - "phf_macros", "phf_shared 0.10.0", - "proc-macro-hack", +] + +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared 0.11.2", ] [[package]] @@ -2045,16 +2015,6 @@ dependencies = [ "phf_shared 0.7.24", ] -[[package]] -name = "phf_codegen" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" -dependencies = [ - "phf_generator 0.8.0", - "phf_shared 0.8.0", -] - [[package]] name = "phf_codegen" version = "0.10.0" @@ -2075,16 +2035,6 @@ dependencies = [ "rand 0.6.5", ] -[[package]] -name = "phf_generator" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" -dependencies = [ - "phf_shared 0.8.0", - "rand 0.7.3", -] - [[package]] name = "phf_generator" version = "0.10.0" @@ -2096,17 +2046,26 @@ dependencies = [ ] [[package]] -name = "phf_macros" -version = "0.10.0" +name = "phf_generator" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", - "proc-macro-hack", - "proc-macro2 1.0.60", - "quote 1.0.28", - "syn 1.0.109", + "phf_shared 0.11.2", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator 0.11.2", + "phf_shared 0.11.2", + "proc-macro2 1.0.64", + "quote 1.0.29", + "syn 2.0.26", ] [[package]] @@ -2118,15 +2077,6 @@ dependencies = [ "siphasher 0.2.3", ] -[[package]] -name = "phf_shared" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" -dependencies = [ - "siphasher 0.3.10", -] - [[package]] name = "phf_shared" version = "0.10.0" @@ -2137,10 +2087,19 @@ dependencies = [ ] [[package]] -name = "pin-project-lite" -version = "0.2.9" +name = "phf_shared" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher 0.3.10", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" [[package]] name = "pin-utils" @@ -2194,12 +2153,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "0.4.30" @@ -2211,9 +2164,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" dependencies = [ "unicode-ident", ] @@ -2239,11 +2192,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" dependencies = [ - "proc-macro2 1.0.60", + "proc-macro2 1.0.64", ] [[package]] @@ -2256,29 +2209,15 @@ dependencies = [ "libc", "rand_chacha 0.1.1", "rand_core 0.4.2", - "rand_hc 0.1.0", + "rand_hc", "rand_isaac", "rand_jitter", "rand_os", - "rand_pcg 0.1.2", + "rand_pcg", "rand_xorshift", "winapi 0.3.9", ] -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc 0.2.0", - "rand_pcg 0.2.1", -] - [[package]] name = "rand" version = "0.8.5" @@ -2300,16 +2239,6 @@ dependencies = [ "rand_core 0.3.1", ] -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - [[package]] name = "rand_chacha" version = "0.3.1" @@ -2335,22 +2264,13 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", -] - [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", + "getrandom", ] [[package]] @@ -2362,15 +2282,6 @@ dependencies = [ "rand_core 0.3.1", ] -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "rand_isaac" version = "0.1.1" @@ -2415,15 +2326,6 @@ dependencies = [ "rand_core 0.4.2", ] -[[package]] -name = "rand_pcg" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "rand_xorshift" version = "0.1.1" @@ -2471,7 +2373,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f" dependencies = [ "combine", - "itoa 1.0.6", + "itoa 1.0.8", "percent-encoding 2.3.0", "ryu", "sha1_smol", @@ -2490,14 +2392,26 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.8.4" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" dependencies = [ "aho-corasick", "memchr", @@ -2506,9 +2420,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "reqwest" @@ -2555,10 +2469,10 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.3.19", + "h2 0.3.20", "http 0.2.9", "http-body 0.4.5", - "hyper 0.14.26", + "hyper 0.14.27", "hyper-tls 0.5.0", "ipnet", "js-sys", @@ -2571,7 +2485,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.1", - "tokio 1.28.2", + "tokio 1.29.1", "tokio-native-tls", "tower-service", "url 2.4.0", @@ -2583,11 +2497,11 @@ dependencies = [ [[package]] name = "rlua" -version = "0.19.5" +version = "0.19.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "753540fb29c9a615ce5dd64be5c957271adc887c8424555fc372e0b88ebde074" +checksum = "5d33e5ba15c3d43178f283ed5863d4531e292fc0e56fb773f3bea45f18e3a42a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bstr", "libc", "num-traits", @@ -2596,9 +2510,9 @@ dependencies = [ [[package]] name = "rlua-lua54-sys" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b1af7df13ef18849005d82d8d16411ebdb7f35f9dcfcb790a4ce49bc1e05aa" +checksum = "7aafabafe1895cb4a2be81a56d7ff3d46bf4b5d2f9cfdbea2ed404cdabe96474" dependencies = [ "cc", "libc", @@ -2631,16 +2545,29 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.20" +version = "0.37.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys", - "windows-sys 0.48.0", + "linux-raw-sys 0.3.8", + "windows-sys", +] + +[[package]] +name = "rustix" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +dependencies = [ + "bitflags 2.3.3", + "errno", + "libc", + "linux-raw-sys 0.4.3", + "windows-sys", ] [[package]] @@ -2657,9 +2584,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" [[package]] name = "same-file" @@ -2672,11 +2599,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -2687,9 +2614,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scraper" -version = "0.16.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59e25654b5e9fd557a67dbaab5a5d36b8c448d0561beb4c041b6dbb902eddfa6" +checksum = "c95a930e03325234c18c7071fd2b60118307e025d6fff3e12745ffbf63a3d29c" dependencies = [ "ahash 0.8.3", "cssparser", @@ -2698,7 +2625,7 @@ dependencies = [ "html5ever 0.26.0", "once_cell", "selectors", - "smallvec 1.10.0", + "smallvec 1.11.0", "tendril", ] @@ -2708,7 +2635,7 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -2737,20 +2664,21 @@ dependencies = [ [[package]] name = "selectors" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c37578180969d00692904465fb7f6b3d50b9a2b952b87c23d0e2e5cb5013416" +checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" dependencies = [ - "bitflags", + "bitflags 2.3.3", "cssparser", "derive_more", "fxhash", "log", - "phf 0.8.0", - "phf_codegen 0.8.0", + "new_debug_unreachable", + "phf 0.10.1", + "phf_codegen 0.10.0", "precomputed-hash", "servo_arc", - "smallvec 1.10.0", + "smallvec 1.11.0", ] [[package]] @@ -2776,31 +2704,31 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.164" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ - "proc-macro2 1.0.60", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.64", + "quote 1.0.29", + "syn 2.0.26", ] [[package]] name = "serde_json" -version = "1.0.97" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" +checksum = "b5062a995d481b2308b6064e9af76011f2921c35f97b0468811ed9f6cd91dfed" dependencies = [ - "itoa 1.0.6", + "itoa 1.0.8", "ryu", "serde", ] @@ -2824,18 +2752,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.6", + "itoa 1.0.8", "ryu", "serde", ] [[package]] name = "servo_arc" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52aa42f8fdf0fed91e5ce7f23d8138441002fa31dca008acf47e6fd4721f741" +checksum = "d036d71a959e00c77a63538b90a6c2390969f9772b096ea837205c6bd0491a44" dependencies = [ - "nodrop", "stable_deref_trait", ] @@ -2908,9 +2835,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "socket2" @@ -2974,8 +2901,8 @@ checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" dependencies = [ "phf_generator 0.7.24", "phf_shared 0.7.24", - "proc-macro2 1.0.60", - "quote 1.0.28", + "proc-macro2 1.0.64", + "quote 1.0.29", "string_cache_shared", ] @@ -2987,8 +2914,8 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", - "proc-macro2 1.0.60", - "quote 1.0.28", + "proc-macro2 1.0.64", + "quote 1.0.29", ] [[package]] @@ -3014,19 +2941,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.60", - "quote 1.0.28", + "proc-macro2 1.0.64", + "quote 1.0.29", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.18" +version = "2.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" dependencies = [ - "proc-macro2 1.0.60", - "quote 1.0.28", + "proc-macro2 1.0.64", + "quote 1.0.29", "unicode-ident", ] @@ -3036,8 +2963,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.60", - "quote 1.0.28", + "proc-macro2 1.0.64", + "quote 1.0.29", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -3052,8 +2979,8 @@ dependencies = [ "cfg-if 1.0.0", "fastrand", "redox_syscall 0.3.5", - "rustix", - "windows-sys 0.48.0", + "rustix 0.37.23", + "windows-sys", ] [[package]] @@ -3078,22 +3005,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ - "proc-macro2 1.0.60", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.64", + "quote 1.0.29", + "syn 2.0.26", ] [[package]] @@ -3109,11 +3036,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" +checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" dependencies = [ - "itoa 1.0.6", + "itoa 1.0.8", "serde", "time-core", "time-macros", @@ -3127,9 +3054,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" dependencies = [ "time-core", ] @@ -3180,11 +3107,12 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.2" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg 1.1.0", + "backtrace", "bytes 1.4.0", "libc", "mio 0.8.8", @@ -3194,7 +3122,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -3245,9 +3173,9 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2 1.0.60", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.64", + "quote 1.0.29", + "syn 2.0.26", ] [[package]] @@ -3257,7 +3185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.28.2", + "tokio 1.29.1", ] [[package]] @@ -3342,7 +3270,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.28.2", + "tokio 1.29.1", "tracing", ] @@ -3405,9 +3333,9 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "ucd-trie" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] name = "unicase" @@ -3426,9 +3354,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -3536,12 +3464,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -3573,9 +3495,9 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.60", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.64", + "quote 1.0.29", + "syn 2.0.26", "wasm-bindgen-shared", ] @@ -3597,7 +3519,7 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ - "quote 1.0.28", + "quote 1.0.29", "wasm-bindgen-macro-support", ] @@ -3607,9 +3529,9 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ - "proc-macro2 1.0.60", - "quote 1.0.28", - "syn 2.0.18", + "proc-macro2 1.0.64", + "quote 1.0.29", + "syn 2.0.26", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3632,7 +3554,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.13.0" +version = "0.14.0" dependencies = [ "actix-files", "actix-web", @@ -3653,7 +3575,7 @@ dependencies = [ "scraper", "serde", "serde_json", - "tokio 1.28.2", + "tokio 1.29.1", ] [[package]] @@ -3699,21 +3621,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -3725,97 +3632,55 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 0615637..fe75fb8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.13.0" +version = "0.14.0" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" From cff7de9d0f313a4e1de0b79a7b392ff74c3b7cd8 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 17 Jul 2023 10:47:29 +0300 Subject: [PATCH 20/24] =?UTF-8?q?=E2=9C=A8=20feat:=20rephrase=20the=20comm?= =?UTF-8?q?ent=20to=20break=20it=20into=20smaller=20sentences.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/results/aggregator.rs | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index ec5c065..b06d9ee 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -21,18 +21,23 @@ use crate::engines::{ /// Aliases for long type annotations type FutureVec = Vec, Report>>>; -/// A function that aggregates all the scraped results from the above user selected upstream -/// search engines either selected from the UI or from the config file which is handled by the code -/// by matching over the selected search engines and adding the selected ones to the vector which -/// is then used to create an async task vector with `tokio::spawn` which returns a future which -/// is then awaited on in another loop and then all the collected results is filtered for errors -/// and proper results and if an error is found is then sent to the UI with the engine name and the -/// error type that caused it by putting them finallt in the returned `SearchResults` struct. Also -/// the same process also removes duplicate results and if two results are found to be from two or -/// more engines then puts their names together to show the results are fetched from these upstream -/// engines and then removes all data from the HashMap and puts into a struct of all results aggregated -/// into a vector and also adds the query used into the struct this is neccessory because otherwise the -/// search bar in search remains empty if searched from the query url. +/// The function aggregates the scraped results from the user-selected upstream search engines. +/// These engines can be chosen either from the user interface (UI) or from the configuration file. +/// The code handles this process by matching the selected search engines and adding them to a vector. +/// This vector is then used to create an asynchronous task vector using `tokio::spawn`, which returns +/// a future. This future is awaited in another loop. Once the results are collected, they are filtered +/// to remove any errors and ensure only proper results are included. If an error is encountered, it is +/// sent to the UI along with the name of the engine and the type of error. This information is finally +/// placed in the returned `SearchResults` struct. +/// +/// Additionally, the function eliminates duplicate results. If two results are identified as coming from +/// multiple engines, their names are combined to indicate that the results were fetched from these upstream +/// engines. After this, all the data in the `HashMap` is removed and placed into a struct that contains all +/// the aggregated results in a vector. Furthermore, the query used is also added to the struct. This step is +/// necessary to ensure that the search bar in the search remains populated even when searched from the query URL. +/// +/// Overall, this function serves to aggregate scraped results from user-selected search engines, handling errors, +/// removing duplicates, and organizing the data for display in the UI. /// /// # Example: /// From c5fca3255339509e7a268806de5fb5b82bac9664 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 17 Jul 2023 10:50:15 +0300 Subject: [PATCH 21/24] =?UTF-8?q?=E2=9C=A8=20feat:=20fix=20spelling=20mist?= =?UTF-8?q?ake=20in=20the=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/server/routes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/routes.rs b/src/server/routes.rs index c4e18c2..de9ab77 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -126,7 +126,7 @@ async fn results( let mut redis_cache = RedisCache::new(config.redis_url.clone())?; // fetch the cached results json. let cached_results_json = redis_cache.cached_json(&url); - // check if fetched catch results was indeed fetched or it was an error and if so + // check if fetched cache results was indeed fetched or it was an error and if so // handle the data accordingly. match cached_results_json { Ok(results_json) => Ok(serde_json::from_str::(&results_json).unwrap()), From 7e3b59e8c0ac5d7629a25cc35cc0fbba3f16aa50 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 17 Jul 2023 13:03:20 +0300 Subject: [PATCH 22/24] =?UTF-8?q?=E2=9C=A8=20feat:=20remove=20the=20empty?= =?UTF-8?q?=5Fresult=5Fset=20option=20and=20helper=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/results/aggregation_models.rs | 12 ------------ src/server/routes.rs | 6 ------ 2 files changed, 18 deletions(-) diff --git a/src/results/aggregation_models.rs b/src/results/aggregation_models.rs index 307cef9..6766fae 100644 --- a/src/results/aggregation_models.rs +++ b/src/results/aggregation_models.rs @@ -155,7 +155,6 @@ pub struct SearchResults { pub page_query: String, pub style: Style, pub engine_errors_info: Vec, - pub empty_result_set: bool, } impl SearchResults { @@ -179,7 +178,6 @@ impl SearchResults { page_query, style: Style::new("".to_string(), "".to_string()), engine_errors_info, - empty_result_set: false, } } @@ -187,14 +185,4 @@ impl SearchResults { pub fn add_style(&mut self, style: Style) { self.style = style; } - - /// A function which checks whether the results stored are empty or not. - pub fn is_empty_result_set(&self) -> bool { - self.results.is_empty() - } - - /// A setter function which sets the empty_result_set to true. - pub fn set_empty_result_set(&mut self) { - self.empty_result_set = true; - } } diff --git a/src/server/routes.rs b/src/server/routes.rs index de9ab77..e846fd9 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -161,12 +161,6 @@ async fn results( } }; results_json.add_style(config.style.clone()); - // check whether the results grabbed from the upstream engines are empty or - // not if they are empty then set the empty_result_set option to true in - // the result json. - if results_json.is_empty_result_set() { - results_json.set_empty_result_set(); - } redis_cache.cache_results(serde_json::to_string(&results_json)?, &url)?; Ok(results_json) } From 987e667c3d2697579ca57cc8cf855aeca36b90b1 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 17 Jul 2023 13:17:24 +0300 Subject: [PATCH 23/24] =?UTF-8?q?=E2=9C=A8=20feat:=20rename=20`results=5Fj?= =?UTF-8?q?son`=20to=20`results`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/server/routes.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server/routes.rs b/src/server/routes.rs index e846fd9..cb6999d 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -129,13 +129,13 @@ async fn results( // check if fetched cache results was indeed fetched or it was an error and if so // handle the data accordingly. match cached_results_json { - Ok(results_json) => Ok(serde_json::from_str::(&results_json).unwrap()), + Ok(results) => Ok(serde_json::from_str::(&results).unwrap()), Err(_) => { // check if the cookie value is empty or not if it is empty then use the // default selected upstream search engines from the config file otherwise // parse the non-empty cookie and grab the user selected engines from the // UI and use that. - let mut results_json: crate::results::aggregation_models::SearchResults = match req + let mut results: crate::results::aggregation_models::SearchResults = match req .cookie("appCookie") { Some(cookie_value) => { @@ -160,9 +160,9 @@ async fn results( .await? } }; - results_json.add_style(config.style.clone()); - redis_cache.cache_results(serde_json::to_string(&results_json)?, &url)?; - Ok(results_json) + results.add_style(config.style.clone()); + redis_cache.cache_results(serde_json::to_string(&results)?, &url)?; + Ok(results) } } } From a28d5594269a4c9e45c08a4e9db09139b5f23eb3 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 17 Jul 2023 17:03:50 +0300 Subject: [PATCH 24/24] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20documentation?= =?UTF-8?q?=20for=20the=20`outputs.iter()`=20code=20block?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/results/aggregator.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index b06d9ee..501b273 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -108,6 +108,27 @@ pub async fn aggregate( let mut engine_errors_info: Vec = Vec::new(); + // The code block `outputs.iter()` determines whether it is the first time the code is being run. + // It does this by checking the initial flag. If it is the first time, the code selects the first + // engine from which results are fetched and adds or extends them into the `result_map`. If the + // initially selected engine fails, the code automatically selects another engine to map or extend + // into the `result_map`. On the other hand, if an engine selected for the first time successfully + // fetches results and maps them into the `result_map`, the initial flag is set to false. Subsequently, + // the code iterates through the remaining engines one by one. It compares the fetched results from each + // engine with the results already present in the `result_map` to identify any duplicates. If duplicate + // results are found, the code groups them together with the name of the engine from which they were + // fetched, and automatically removes the duplicate results from the newly fetched data. + // + // Additionally, the code handles errors returned by the engines. It keeps track of which engines + // encountered errors and stores this information in a vector of structures called `EngineErrorInfo`. + // Each structure in this vector contains the name of the engine and the type of error it returned. + // These structures will later be added to the final `SearchResults` structure. The `SearchResults` + // structure is used to display an error box in the UI containing the relevant information from + // the `EngineErrorInfo` structure. + // + // In summary, this code block manages the selection of engines, handling of duplicate results, and tracking + // of errors in order to populate the `result_map` and provide informative feedback to the user through the + // `SearchResults` structure. let mut initial: bool = true; let mut counter: usize = 0; outputs.iter().for_each(|results| {