diff --git a/Cargo.lock b/Cargo.lock index 37e0c2e..2aa901c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,12 +83,12 @@ dependencies = [ [[package]] name = "actix-macros" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" +checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ - "quote 1.0.31", - "syn 1.0.109", + "quote 1.0.32", + "syn 2.0.27", ] [[package]] @@ -202,7 +202,7 @@ checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" dependencies = [ "actix-router", "proc-macro2 1.0.66", - "quote 1.0.31", + "quote 1.0.32", "syn 1.0.109", ] @@ -294,13 +294,13 @@ checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" [[package]] name = "async-trait" -version = "0.1.71" +version = "0.1.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" +checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.31", - "syn 2.0.26", + "quote 1.0.32", + "syn 2.0.27", ] [[package]] @@ -517,18 +517,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.15" +version = "4.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f644d0dac522c8b05ddc39aaaccc5b136d5dc4ff216610c5641e3be5becf56c" +checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.3.15" +version = "4.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af410122b9778e024f9e0fb35682cc09cc3f85cad5e8d3ba8f47a9702df6e73d" +checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1" dependencies = [ "anstyle", "clap_lex", @@ -794,8 +794,8 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ - "quote 1.0.31", - "syn 2.0.26", + "quote 1.0.32", + "syn 2.0.27", ] [[package]] @@ -806,7 +806,7 @@ checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", "proc-macro2 1.0.66", - "quote 1.0.31", + "quote 1.0.32", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -850,9 +850,9 @@ checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "encoding_rs" @@ -888,9 +888,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", @@ -934,7 +934,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.31", + "quote 1.0.32", "syn 1.0.109", "synstructure", ] @@ -952,12 +952,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "flate2" @@ -1231,7 +1228,7 @@ dependencies = [ "mac", "markup5ever 0.11.0", "proc-macro2 1.0.66", - "quote 1.0.31", + "quote 1.0.32", "syn 1.0.109", ] @@ -1426,26 +1423,6 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] - [[package]] name = "iovec" version = "0.1.4" @@ -1468,7 +1445,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.4", + "rustix", "windows-sys", ] @@ -1539,12 +1516,6 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" -[[package]] -name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - [[package]] name = "linux-raw-sys" version = "0.4.3" @@ -1784,9 +1755,9 @@ checksum = "ab250442c86f1850815b5d268639dff018c0627022bc1940eb2d642ca1ce12f0" [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg 1.1.0", ] @@ -1844,8 +1815,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.31", - "syn 2.0.26", + "quote 1.0.32", + "syn 2.0.27", ] [[package]] @@ -1962,8 +1933,8 @@ dependencies = [ "pest", "pest_meta", "proc-macro2 1.0.66", - "quote 1.0.31", - "syn 2.0.26", + "quote 1.0.32", + "syn 2.0.27", ] [[package]] @@ -2064,8 +2035,8 @@ dependencies = [ "phf_generator 0.11.2", "phf_shared 0.11.2", "proc-macro2 1.0.66", - "quote 1.0.31", - "syn 2.0.26", + "quote 1.0.32", + "syn 2.0.27", ] [[package]] @@ -2192,9 +2163,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.31" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2 1.0.66", ] @@ -2368,15 +2339,16 @@ dependencies = [ [[package]] name = "redis" -version = "0.23.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f" +checksum = "ff5d95dd18a4d76650f0c2607ed8ebdbf63baf9cb934e1c233cd220c694db1d7" dependencies = [ "combine", "itoa 1.0.9", "percent-encoding 2.3.0", "ryu", "sha1_smol", + "socket2", "url 2.4.0", ] @@ -2409,9 +2381,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" +checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294" dependencies = [ "aho-corasick", "memchr", @@ -2543,20 +2515,6 @@ dependencies = [ "semver 1.0.18", ] -[[package]] -name = "rustix" -version = "0.37.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys", -] - [[package]] name = "rustix" version = "0.38.4" @@ -2566,7 +2524,7 @@ dependencies = [ "bitflags 2.3.3", "errno", "libc", - "linux-raw-sys 0.4.3", + "linux-raw-sys", "windows-sys", ] @@ -2631,9 +2589,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -2644,9 +2602,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" dependencies = [ "core-foundation-sys", "libc", @@ -2704,29 +2662,29 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.171" +version = "1.0.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "60363bdd39a7be0266a520dab25fdc9241d2f987b08a01e01f0ec6d06a981348" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "f28482318d6641454cb273da158647922d1be6b5a2fcc6165cd89ebdd7ed576b" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.31", - "syn 2.0.26", + "quote 1.0.32", + "syn 2.0.27", ] [[package]] name = "serde_json" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ "itoa 1.0.9", "ryu", @@ -2902,7 +2860,7 @@ dependencies = [ "phf_generator 0.7.24", "phf_shared 0.7.24", "proc-macro2 1.0.66", - "quote 1.0.31", + "quote 1.0.32", "string_cache_shared", ] @@ -2915,7 +2873,7 @@ dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", "proc-macro2 1.0.66", - "quote 1.0.31", + "quote 1.0.32", ] [[package]] @@ -2942,18 +2900,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.31", + "quote 1.0.32", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.26" +version = "2.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" +checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.31", + "quote 1.0.32", "unicode-ident", ] @@ -2964,22 +2922,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.31", + "quote 1.0.32", "syn 1.0.109", "unicode-xid 0.2.4", ] [[package]] name = "tempfile" -version = "3.6.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" dependencies = [ - "autocfg 1.1.0", "cfg-if 1.0.0", "fastrand", "redox_syscall 0.3.5", - "rustix 0.37.23", + "rustix", "windows-sys", ] @@ -3005,22 +2962,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.31", - "syn 2.0.26", + "quote 1.0.32", + "syn 2.0.27", ] [[package]] @@ -3174,8 +3131,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.31", - "syn 2.0.26", + "quote 1.0.32", + "syn 2.0.27", ] [[package]] @@ -3496,8 +3453,8 @@ dependencies = [ "log", "once_cell", "proc-macro2 1.0.66", - "quote 1.0.31", - "syn 2.0.26", + "quote 1.0.32", + "syn 2.0.27", "wasm-bindgen-shared", ] @@ -3519,7 +3476,7 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ - "quote 1.0.31", + "quote 1.0.32", "wasm-bindgen-macro-support", ] @@ -3530,8 +3487,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.31", - "syn 2.0.26", + "quote 1.0.32", + "syn 2.0.27", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3554,7 +3511,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.14.1" +version = "0.15.0" dependencies = [ "actix-files", "actix-web", @@ -3717,18 +3674,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.3+zstd.1.5.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.5+zstd.1.5.4" +version = "6.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" +checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" dependencies = [ "libc", "zstd-sys", diff --git a/Cargo.toml b/Cargo.toml index 5ba065f..3bd7350 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.14.1" +version = "0.15.0" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" diff --git a/src/config/parser.rs b/src/config/parser.rs index 971cd48..a665246 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -22,6 +22,7 @@ static CONFIG_FILE_NAME: &str = "config.lua"; /// * `logging` - It stores the option to whether enable or disable logs. /// * `debug` - It stores the option to whether enable or disable debug mode. /// * `upstream_search_engines` - It stores all the engine names that were enabled by the user. +/// * `request_timeout` - It stores the time (secs) which controls the server request timeout. #[derive(Clone)] pub struct Config { pub port: u16, @@ -32,6 +33,7 @@ pub struct Config { pub logging: bool, pub debug: bool, pub upstream_search_engines: Vec, + pub request_timeout: u8, } /// Configuration options for the aggregator. @@ -80,6 +82,7 @@ impl Config { .into_iter() .filter_map(|(key, value)| value.then_some(key)) .collect(), + request_timeout: globals.get::<_, u8>("request_timeout")?, }) }) } diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index f8ad597..487e120 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -29,6 +29,7 @@ impl SearchEngine for DuckDuckGo { /// * `query` - Takes the user provided query to query to the upstream search engine with. /// * `page` - Takes an u32 as an argument. /// * `user_agent` - Takes a random user agent string as an argument. + /// * `request_timeout` - Takes a time (secs) as a value which controls the server request timeout. /// /// # Errors /// @@ -41,6 +42,7 @@ impl SearchEngine for DuckDuckGo { query: String, page: u32, user_agent: String, + request_timeout: u8, ) -> Result, EngineError> { // Page number can be missing or empty string and so appropriate handling is required // so that upstream server recieves valid page number. @@ -90,7 +92,7 @@ impl SearchEngine for DuckDuckGo { ); let document: Html = Html::parse_document( - &DuckDuckGo::fetch_html_from_upstream(self, url, header_map).await?, + &DuckDuckGo::fetch_html_from_upstream(self, url, header_map, request_timeout).await?, ); let no_result: Selector = Selector::parse(".no-results") diff --git a/src/engines/engine_models.rs b/src/engines/engine_models.rs index f635ca9..b5051be 100644 --- a/src/engines/engine_models.rs +++ b/src/engines/engine_models.rs @@ -50,11 +50,12 @@ pub trait SearchEngine { &self, url: String, header_map: reqwest::header::HeaderMap, + request_timeout: u8, ) -> Result { // fetch the html from upstream search engine Ok(reqwest::Client::new() .get(url) - .timeout(Duration::from_secs(30)) // Add timeout to request to avoid DDOSing the server + .timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server .headers(header_map) // add spoofed headers to emulate human behaviour .send() .await @@ -71,5 +72,6 @@ pub trait SearchEngine { query: String, page: u32, user_agent: String, + request_timeout: u8, ) -> Result, EngineError>; } diff --git a/src/engines/searx.rs b/src/engines/searx.rs index 145abf1..5fd7354 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -27,6 +27,7 @@ impl SearchEngine for Searx { /// * `query` - Takes the user provided query to query to the upstream search engine with. /// * `page` - Takes an u32 as an argument. /// * `user_agent` - Takes a random user agent string as an argument. + /// * `request_timeout` - Takes a time (secs) as a value which controls the server request timeout. /// /// # Errors /// @@ -40,6 +41,7 @@ impl SearchEngine for Searx { query: String, page: u32, user_agent: String, + request_timeout: u8, ) -> Result, EngineError> { // Page number can be missing or empty string and so appropriate handling is required // so that upstream server recieves valid page number. @@ -70,8 +72,9 @@ impl SearchEngine for Searx { ); header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse().into_report().change_context(EngineError::UnexpectedError)?); - let document: Html = - Html::parse_document(&Searx::fetch_html_from_upstream(self, url, header_map).await?); + let document: Html = Html::parse_document( + &Searx::fetch_html_from_upstream(self, url, header_map, request_timeout).await?, + ); let no_result: Selector = Selector::parse("#urls>.dialog-error>p") .map_err(|_| Report::new(EngineError::UnexpectedError)) diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index 501b273..b8d7346 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -51,6 +51,7 @@ type FutureVec = Vec, Report< /// * `random_delay` - Accepts a boolean value to add a random delay before making the request. /// * `debug` - Accepts a boolean value to enable or disable debug mode option. /// * `upstream_search_engines` - Accepts a vector of search engine names which was selected by the +/// * `request_timeout` - Accepts a time (secs) as a value which controls the server request timeout. /// user through the UI or the config file. /// /// # Error @@ -64,6 +65,7 @@ pub async fn aggregate( random_delay: bool, debug: bool, upstream_search_engines: Vec, + request_timeout: u8, ) -> Result> { let user_agent: String = random_user_agent(); let mut result_map: HashMap = HashMap::new(); @@ -92,9 +94,11 @@ pub async fn aggregate( .map(|search_engine| { let query: String = query.clone(); let user_agent: String = user_agent.clone(); - tokio::spawn( - async move { search_engine.results(query, page, user_agent.clone()).await }, - ) + tokio::spawn(async move { + search_engine + .results(query, page, user_agent.clone(), request_timeout) + .await + }) }) .collect(); diff --git a/src/server/routes.rs b/src/server/routes.rs index cb6999d..0b07115 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -146,6 +146,7 @@ async fn results( config.aggregator.random_delay, config.debug, cookie_value.engines, + config.request_timeout, ) .await? } @@ -156,6 +157,7 @@ async fn results( config.aggregator.random_delay, config.debug, config.upstream_search_engines.clone(), + config.request_timeout, ) .await? } diff --git a/websurfx/config.lua b/websurfx/config.lua index b96bf2f..73bffe7 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -8,6 +8,7 @@ binding_ip = "127.0.0.1" --ip address on the which server should be launched. production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users (more than one)) -- if production_use is set to true -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. +request_timeout = 30 -- timeout for the search requests sent to the upstream search engines to be fetched (value in seconds). -- ### Website ### -- The different colorschemes provided are: