From 019b3322e79b5e549bf12222aa4964c43dae3154 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 16 May 2023 12:22:00 +0300 Subject: [PATCH 1/5] chore: add enum to handle different reqwest errors & add timeout to requests --- src/engines/duckduckgo.rs | 3 ++- src/engines/engine_models.rs | 8 ++++++++ src/engines/mod.rs | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 src/engines/engine_models.rs diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index 254ab16..7232165 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -2,7 +2,7 @@ //! by querying the upstream duckduckgo search engine with user provided query and with a page //! number if provided. -use std::collections::HashMap; +use std::{collections::HashMap, time::Duration}; use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT}; use scraper::{Html, Selector}; @@ -57,6 +57,7 @@ pub async fn results( // TODO: Write better error handling code to handle no results case. let results: String = reqwest::Client::new() .get(url) + .timeout(Duration::from_secs(30)) .headers(header_map) // add spoofed headers to emulate human behaviour .send() .await? diff --git a/src/engines/engine_models.rs b/src/engines/engine_models.rs new file mode 100644 index 0000000..7dc8ad6 --- /dev/null +++ b/src/engines/engine_models.rs @@ -0,0 +1,8 @@ +#[derive(Debug)] +pub enum ReqwestError{ + NotFound, + Timeout, + Forbidden, + AccessDenied, + TooManyRequests +} diff --git a/src/engines/mod.rs b/src/engines/mod.rs index 7f390b1..f9bb8ad 100644 --- a/src/engines/mod.rs +++ b/src/engines/mod.rs @@ -1,2 +1,3 @@ pub mod duckduckgo; +pub mod engine_models; pub mod searx; From 5962cca294df604d093f9e82ab7ab5c52e73ad37 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 31 May 2023 19:54:51 +0300 Subject: [PATCH 2/5] chore: provide a better and more standardized way to handle engine errors --- src/engines/duckduckgo.rs | 19 +++++++--- src/engines/engine_models.rs | 70 ++++++++++++++++++++++++++++++++---- src/engines/searx.rs | 23 +++++++++--- 3 files changed, 96 insertions(+), 16 deletions(-) diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index 7232165..219801a 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -9,6 +9,8 @@ use scraper::{Html, Selector}; use crate::search_results_handler::aggregation_models::RawSearchResult; +use super::engine_models::EngineErrorKind; + /// This function scrapes results from the upstream engine duckduckgo and puts all the scraped /// results like title, visiting_url (href in html),engine (from which engine it was fetched from) /// and description in a RawSearchResult and then adds that to HashMap whose keys are url and @@ -22,14 +24,15 @@ use crate::search_results_handler::aggregation_models::RawSearchResult; /// /// # Errors /// -/// Returns a reqwest error if the user is not connected to the internet or if their is failure to -/// reach the above `upstream search engine` page and also returns error if the scraping -/// selector fails to initialize" +/// Returns an `EngineErrorKind` if the user is not connected to the internet or if their is failure to +/// reach the above `upstream search engine` page or if the `upstream search engine` is unable to +/// provide results for the requested search query and also returns error if the scraping selector +/// or HeaderMap fails to initialize. pub async fn results( query: &str, page: u32, user_agent: &str, -) -> Result, Box> { +) -> Result, EngineErrorKind> { // Page number can be missing or empty string and so appropriate handling is required // so that upstream server recieves valid page number. let url: String = match page { @@ -54,7 +57,6 @@ pub async fn results( header_map.insert(COOKIE, "kl=wt-wt".parse()?); // fetch the html from upstream duckduckgo engine - // TODO: Write better error handling code to handle no results case. let results: String = reqwest::Client::new() .get(url) .timeout(Duration::from_secs(30)) @@ -65,6 +67,13 @@ pub async fn results( .await?; let document: Html = Html::parse_document(&results); + + let no_result: Selector = Selector::parse(".no-results")?; + + if let Some(_) = document.select(&no_result).next() { + return Err(EngineErrorKind::EmptyResultSet); + } + let results: Selector = Selector::parse(".result")?; let result_title: Selector = Selector::parse(".result__a")?; let result_url: Selector = Selector::parse(".result__url")?; diff --git a/src/engines/engine_models.rs b/src/engines/engine_models.rs index 7dc8ad6..16cc266 100644 --- a/src/engines/engine_models.rs +++ b/src/engines/engine_models.rs @@ -1,8 +1,66 @@ +//! This module provides the error enum to handle different errors associated while requesting data from +//! the upstream search engines with the search query provided by the user. + +use reqwest::header::InvalidHeaderValue; +use scraper::error::SelectorErrorKind; + +/// A custom error type used for handle engine associated errors. +/// +/// This enum provides variants three different categories of errors: +/// * `RequestError` - This variant handles all request related errors like forbidden, not found, +/// etc. +/// * `EmptyResultSet` - This variant handles the not results found error provide by the upstream +/// search engines. +/// * `UnexpectedError` - This variant handles all the errors which are unexpected or occur rarely +/// and are errors mostly related to failure in initialization of HeaderMap, Selector errors and +/// all other errors. #[derive(Debug)] -pub enum ReqwestError{ - NotFound, - Timeout, - Forbidden, - AccessDenied, - TooManyRequests +pub enum EngineErrorKind { + RequestError(reqwest::Error), + EmptyResultSet, + UnexpectedError(String), +} + +/// Implementing `Display` trait to make errors writable on the stdout and also providing/passing the +/// appropriate errors that should be written to the stdout when this error is raised/encountered. +impl std::fmt::Display for EngineErrorKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EngineErrorKind::RequestError(request_error) => write!(f, "{}", request_error), + EngineErrorKind::EmptyResultSet => { + write!(f, "The upstream search engine returned an empty result set") + } + EngineErrorKind::UnexpectedError(unexpected_error) => write!(f, "{}", unexpected_error), + } + } +} + +/// Implementing `Error` trait to make the the `EngineErrorKind` enum an error type. +impl std::error::Error for EngineErrorKind {} + +/// Implementing `From` trait to map the `SelectorErrorKind` to `UnexpectedError` variant. +impl<'a> From> for EngineErrorKind { + fn from(err: SelectorErrorKind<'a>) -> Self { + match err { + _ => Self::UnexpectedError(err.to_string()), + } + } +} + +/// Implementing `From` trait to map the `InvalidHeaderValue` to `UnexpectedError` variant. +impl<'a> From for EngineErrorKind { + fn from(err: InvalidHeaderValue) -> Self { + match err { + _ => Self::UnexpectedError(err.to_string()), + } + } +} + +/// Implementing `From` trait to map all `reqwest::Error` to `UnexpectedError` variant. +impl<'a> From for EngineErrorKind { + fn from(err: reqwest::Error) -> Self { + match err { + _ => Self::RequestError(err), + } + } } diff --git a/src/engines/searx.rs b/src/engines/searx.rs index 8812dd9..feab464 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -8,6 +8,8 @@ use std::collections::HashMap; use crate::search_results_handler::aggregation_models::RawSearchResult; +use super::engine_models::EngineErrorKind; + /// This function scrapes results from the upstream engine duckduckgo and puts all the scraped /// results like title, visiting_url (href in html),engine (from which engine it was fetched from) /// and description in a RawSearchResult and then adds that to HashMap whose keys are url and @@ -21,14 +23,15 @@ use crate::search_results_handler::aggregation_models::RawSearchResult; /// /// # Errors /// -/// Returns a reqwest error if the user is not connected to the internet or if their is failure to -/// reach the above `upstream search engine` page and also returns error if the scraping -/// selector fails to initialize" +/// Returns an `EngineErrorKind` if the user is not connected to the internet or if their is failure to +/// reach the above `upstream search engine` page or if the `upstream search engine` is unable to +/// provide results for the requested search query and also returns error if the scraping selector +/// or HeaderMap fails to initialize. pub async fn results( query: &str, page: u32, user_agent: &str, -) -> Result, Box> { +) -> Result, EngineErrorKind> { // Page number can be missing or empty string and so appropriate handling is required // so that upstream server recieves valid page number. let url: String = format!("https://searx.work/search?q={query}&pageno={page}"); @@ -41,7 +44,6 @@ pub async fn results( header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse()?); // fetch the html from upstream searx instance engine - // TODO: Write better error handling code to handle no results case. let results: String = reqwest::Client::new() .get(url) .headers(header_map) // add spoofed headers to emulate human behaviours. @@ -51,6 +53,17 @@ pub async fn results( .await?; let document: Html = Html::parse_document(&results); + + let no_result: Selector = Selector::parse("#urls>.dialog-error>p")?; + + if let Some(no_result_msg) = document.select(&no_result).nth(1) { + if no_result_msg.inner_html() + == "we didn't find any results. Please use another query or search in more categories" + { + return Err(EngineErrorKind::EmptyResultSet); + } + } + let results: Selector = Selector::parse(".result")?; let result_title: Selector = Selector::parse("h3>a")?; let result_url: Selector = Selector::parse("h3>a")?; From 5b4a6a82829312abdb5c177c0a227cd65f15772a Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 31 May 2023 20:14:26 +0300 Subject: [PATCH 3/5] chore: bump version to v0.12.0 --- Cargo.lock | 277 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 138 insertions(+), 141 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85310e9..ccc0f05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,19 +4,19 @@ version = 3 [[package]] name = "actix-codec" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" +checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ "bitflags", "bytes 1.4.0", "futures-core", "futures-sink", - "log", "memchr", "pin-project-lite", - "tokio 1.28.0", + "tokio 1.28.2", "tokio-util", + "tracing", ] [[package]] @@ -53,7 +53,7 @@ dependencies = [ "actix-service", "actix-utils", "ahash 0.8.3", - "base64 0.21.0", + "base64 0.21.2", "bitflags", "brotli", "bytes 1.4.0", @@ -62,7 +62,7 @@ dependencies = [ "encoding_rs", "flate2", "futures-core", - "h2 0.3.18", + "h2 0.3.19", "http 0.2.9", "httparse", "httpdate", @@ -75,7 +75,7 @@ dependencies = [ "rand 0.8.5", "sha1", "smallvec 1.10.0", - "tokio 1.28.0", + "tokio 1.28.2", "tokio-util", "tracing", "zstd", @@ -87,7 +87,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "syn 1.0.109", ] @@ -111,7 +111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" dependencies = [ "futures-core", - "tokio 1.28.0", + "tokio 1.28.2", ] [[package]] @@ -125,10 +125,10 @@ dependencies = [ "actix-utils", "futures-core", "futures-util", - "mio 0.8.6", + "mio 0.8.8", "num_cpus", "socket2", - "tokio 1.28.0", + "tokio 1.28.2", "tracing", ] @@ -201,8 +201,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" dependencies = [ "actix-router", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -315,9 +315,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "bit-set" @@ -381,9 +381,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -605,8 +605,8 @@ dependencies = [ "itoa 1.0.6", "matches", "phf 0.10.1", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "smallvec 1.10.0", "syn 1.0.109", ] @@ -617,7 +617,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "syn 1.0.109", ] @@ -628,17 +628,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "rustc_version 0.4.0", "syn 1.0.109", ] [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -730,8 +730,8 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", "synstructure", ] @@ -959,9 +959,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ "bytes 1.4.0", "fnv", @@ -971,16 +971,16 @@ dependencies = [ "http 0.2.9", "indexmap", "slab", - "tokio 1.28.0", + "tokio 1.28.2", "tokio-util", "tracing", ] [[package]] name = "handlebars" -version = "4.3.6" +version = "4.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "035ef95d03713f2c347a72547b7cd38cbc9af7cd51e6099fb62d586d4a6dee3a" +checksum = "83c3372087601b532857d332f5957cbae686da52bb7810bf038c3e3c3cc2fa0d" dependencies = [ "log", "pest", @@ -1035,8 +1035,8 @@ dependencies = [ "log", "mac", "markup5ever 0.11.0", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -1149,7 +1149,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.18", + "h2 0.3.19", "http 0.2.9", "http-body 0.4.5", "httparse", @@ -1157,7 +1157,7 @@ dependencies = [ "itoa 1.0.6", "pin-project-lite", "socket2", - "tokio 1.28.0", + "tokio 1.28.2", "tower-service", "tracing", "want 0.3.0", @@ -1185,7 +1185,7 @@ dependencies = [ "bytes 1.4.0", "hyper 0.14.26", "native-tls", - "tokio 1.28.0", + "tokio 1.28.2", "tokio-native-tls", ] @@ -1242,9 +1242,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -1301,9 +1301,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.62" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68c16e1bfd491478ab155fd8b4896b86f9ede344949b641e61501e07c2b8b4d5" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] @@ -1338,9 +1338,9 @@ checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "linux-raw-sys" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "local-channel" @@ -1381,12 +1381,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" [[package]] name = "mac" @@ -1513,14 +1510,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -1606,15 +1603,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" [[package]] name = "openssl" -version = "0.10.52" +version = "0.10.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" +checksum = "12df40a956736488b7b44fe79fe12d4f245bb5b3f5a1f6095e499760015be392" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -1631,9 +1628,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -1644,9 +1641,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.87" +version = "0.9.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" +checksum = "c2ce0f250f34a308dcfdbb351f511359857d4ed2134ba715a4eadd46e1ffd617" dependencies = [ "cc", "libc", @@ -1749,9 +1746,9 @@ checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -1863,8 +1860,8 @@ dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", "proc-macro-hack", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -1942,9 +1939,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" dependencies = [ "unicode-ident", ] @@ -1970,11 +1967,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.59", ] [[package]] @@ -2213,9 +2210,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.1" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" dependencies = [ "aho-corasick", "memchr", @@ -2224,9 +2221,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "reqwest" @@ -2264,16 +2261,16 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.17" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", "bytes 1.4.0", "encoding_rs", "futures-core", "futures-util", - "h2 0.3.18", + "h2 0.3.19", "http 0.2.9", "http-body 0.4.5", "hyper 0.14.26", @@ -2289,7 +2286,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.1", - "tokio 1.28.0", + "tokio 1.28.2", "tokio-native-tls", "tower-service", "url 2.3.1", @@ -2410,9 +2407,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ "bitflags", "core-foundation", @@ -2423,9 +2420,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" dependencies = [ "core-foundation-sys", "libc", @@ -2482,22 +2479,22 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.162" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b2f6e1ab5c2b98c05f0f35b236b22e8df7ead6ffbf51d7808da7f8817e7ab6" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.162" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2a0814352fd64b58489904a44ea8d90cb1a91dcb6b4f5ebabc32c8318e93cb6" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2680,8 +2677,8 @@ checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" dependencies = [ "phf_generator 0.7.24", "phf_shared 0.7.24", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "string_cache_shared", ] @@ -2693,8 +2690,8 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", ] [[package]] @@ -2720,19 +2717,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.15" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "unicode-ident", ] @@ -2742,8 +2739,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -2796,9 +2793,9 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2875,14 +2872,14 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg 1.1.0", "bytes 1.4.0", "libc", - "mio 0.8.6", + "mio 0.8.8", "num_cpus", "parking_lot 0.12.1", "pin-project-lite", @@ -2940,9 +2937,9 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2952,7 +2949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.28.0", + "tokio 1.28.2", ] [[package]] @@ -3037,7 +3034,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.28.0", + "tokio 1.28.2", "tracing", ] @@ -3061,9 +3058,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", ] @@ -3112,9 +3109,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-normalization" @@ -3243,9 +3240,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b6cb788c4e39112fbe1822277ef6fb3c55cd86b95cb3d3c4c1c9597e4ac74b4" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3253,24 +3250,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e522ed4105a9d626d885b35d62501b30d9666283a5c8be12c14a8bdafe7822" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.35" +version = "0.4.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "083abe15c5d88556b77bdf7aef403625be9e327ad37c62c4e4129af740168163" +checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3280,38 +3277,38 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "358a79a0cb89d21db8120cbfb91392335913e4890665b1a7981d9e956903b434" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4783ce29f09b9d93134d41297aded3a712b7b979e9c6f28c32cb88c973a94869" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a901d592cafaa4d711bc324edfaff879ac700b19c3dfd60058d2b445be2691eb" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "web-sys" -version = "0.3.62" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b5f940c7edfdc6d12126d98c9ef4d1b3d470011c47c76a6581df47ad9ba721" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" dependencies = [ "js-sys", "wasm-bindgen", @@ -3319,7 +3316,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.6.0" +version = "0.12.0" dependencies = [ "actix-files", "actix-web", @@ -3330,12 +3327,12 @@ dependencies = [ "md5", "rand 0.8.5", "redis", - "reqwest 0.11.17", + "reqwest 0.11.18", "rlua", "scraper", "serde", "serde_json", - "tokio 1.28.0", + "tokio 1.28.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ce99ca3..013f709 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.6.0" +version = "0.12.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 4460730ca6ba31b83512dcecc981f49618b9186c Mon Sep 17 00:00:00 2001 From: neon_arch Date: Thu, 1 Jun 2023 12:21:45 +0300 Subject: [PATCH 4/5] chore: make clippy github action happy --- src/config_parser/parser.rs | 2 +- src/engines/duckduckgo.rs | 2 +- src/engines/engine_models.rs | 16 +++++----------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index ac200cd..4c5c1e6 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -118,7 +118,7 @@ impl Config { { Ok("./websurfx/config.lua".to_string()) } else { - Err(format!("Config file not found!!").into()) + Err("Config file not found!!".to_string().into()) } } } diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index 219801a..64c34c3 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -70,7 +70,7 @@ pub async fn results( let no_result: Selector = Selector::parse(".no-results")?; - if let Some(_) = document.select(&no_result).next() { + if document.select(&no_result).next().is_some() { return Err(EngineErrorKind::EmptyResultSet); } diff --git a/src/engines/engine_models.rs b/src/engines/engine_models.rs index 16cc266..3d0a30d 100644 --- a/src/engines/engine_models.rs +++ b/src/engines/engine_models.rs @@ -41,26 +41,20 @@ impl std::error::Error for EngineErrorKind {} /// Implementing `From` trait to map the `SelectorErrorKind` to `UnexpectedError` variant. impl<'a> From> for EngineErrorKind { fn from(err: SelectorErrorKind<'a>) -> Self { - match err { - _ => Self::UnexpectedError(err.to_string()), - } + Self::UnexpectedError(err.to_string()) } } /// Implementing `From` trait to map the `InvalidHeaderValue` to `UnexpectedError` variant. -impl<'a> From for EngineErrorKind { +impl From for EngineErrorKind { fn from(err: InvalidHeaderValue) -> Self { - match err { - _ => Self::UnexpectedError(err.to_string()), - } + Self::UnexpectedError(err.to_string()) } } /// Implementing `From` trait to map all `reqwest::Error` to `UnexpectedError` variant. -impl<'a> From for EngineErrorKind { +impl From for EngineErrorKind { fn from(err: reqwest::Error) -> Self { - match err { - _ => Self::RequestError(err), - } + Self::RequestError(err) } } From c2280b7349eeaf5ede8fed97ecfddc6864bab532 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 4 Jun 2023 11:56:07 +0300 Subject: [PATCH 5/5] chore: add source and an appropriate message to different error variants --- src/engines/engine_models.rs | 47 ++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/src/engines/engine_models.rs b/src/engines/engine_models.rs index 3d0a30d..e763852 100644 --- a/src/engines/engine_models.rs +++ b/src/engines/engine_models.rs @@ -13,12 +13,15 @@ use scraper::error::SelectorErrorKind; /// search engines. /// * `UnexpectedError` - This variant handles all the errors which are unexpected or occur rarely /// and are errors mostly related to failure in initialization of HeaderMap, Selector errors and -/// all other errors. +/// all other errors occuring within the code handling the `upstream search engines`. #[derive(Debug)] pub enum EngineErrorKind { RequestError(reqwest::Error), EmptyResultSet, - UnexpectedError(String), + UnexpectedError { + message: String, + source: Option>, + }, } /// Implementing `Display` trait to make errors writable on the stdout and also providing/passing the @@ -26,29 +29,53 @@ pub enum EngineErrorKind { impl std::fmt::Display for EngineErrorKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - EngineErrorKind::RequestError(request_error) => write!(f, "{}", request_error), + EngineErrorKind::RequestError(request_error) => { + write!(f, "Request error: {}", request_error) + } EngineErrorKind::EmptyResultSet => { write!(f, "The upstream search engine returned an empty result set") } - EngineErrorKind::UnexpectedError(unexpected_error) => write!(f, "{}", unexpected_error), + EngineErrorKind::UnexpectedError { message, source } => { + write!(f, "Unexpected error: {}", message)?; + if let Some(source) = source { + write!(f, "\nCaused by: {}", source)?; + } + Ok(()) + } } } } -/// Implementing `Error` trait to make the the `EngineErrorKind` enum an error type. -impl std::error::Error for EngineErrorKind {} +/// Implementing `Error` trait to make the the `EngineErrorKind` enum an error type and +/// mapping `ReqwestErrors` to `RequestError` and `UnexpectedError` errors to all other unexpected +/// errors ocurring within the code handling the upstream search engines. +impl std::error::Error for EngineErrorKind { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + EngineErrorKind::RequestError(request_error) => Some(request_error), + EngineErrorKind::UnexpectedError { source, .. } => source.as_deref().map(|s| s), + _ => None, + } + } +} /// Implementing `From` trait to map the `SelectorErrorKind` to `UnexpectedError` variant. -impl<'a> From> for EngineErrorKind { - fn from(err: SelectorErrorKind<'a>) -> Self { - Self::UnexpectedError(err.to_string()) +impl From> for EngineErrorKind { + fn from(err: SelectorErrorKind<'_>) -> Self { + Self::UnexpectedError { + message: err.to_string(), + source: None, + } } } /// Implementing `From` trait to map the `InvalidHeaderValue` to `UnexpectedError` variant. impl From for EngineErrorKind { fn from(err: InvalidHeaderValue) -> Self { - Self::UnexpectedError(err.to_string()) + Self::UnexpectedError { + message: err.to_string(), + source: Some(Box::new(err)), + } } }