0
0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-12-22 20:38:22 -05:00

Merge pull request #159 from neon-mmd/feat-add-timeout-opt

 Config option to control the time for server request timeout
This commit is contained in:
zhou fan 2023-07-31 08:51:52 +08:00 committed by GitHub
commit 571b7ce51a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 106 additions and 132 deletions

205
Cargo.lock generated
View File

@ -83,12 +83,12 @@ dependencies = [
[[package]]
name = "actix-macros"
version = "0.2.3"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6"
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
dependencies = [
"quote 1.0.31",
"syn 1.0.109",
"quote 1.0.32",
"syn 2.0.27",
]
[[package]]
@ -202,7 +202,7 @@ checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9"
dependencies = [
"actix-router",
"proc-macro2 1.0.66",
"quote 1.0.31",
"quote 1.0.32",
"syn 1.0.109",
]
@ -294,13 +294,13 @@ checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
[[package]]
name = "async-trait"
version = "0.1.71"
version = "0.1.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf"
checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.31",
"syn 2.0.26",
"quote 1.0.32",
"syn 2.0.27",
]
[[package]]
@ -517,18 +517,18 @@ dependencies = [
[[package]]
name = "clap"
version = "4.3.15"
version = "4.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f644d0dac522c8b05ddc39aaaccc5b136d5dc4ff216610c5641e3be5becf56c"
checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
version = "4.3.15"
version = "4.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af410122b9778e024f9e0fb35682cc09cc3f85cad5e8d3ba8f47a9702df6e73d"
checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1"
dependencies = [
"anstyle",
"clap_lex",
@ -794,8 +794,8 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
dependencies = [
"quote 1.0.31",
"syn 2.0.26",
"quote 1.0.32",
"syn 2.0.27",
]
[[package]]
@ -806,7 +806,7 @@ checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
dependencies = [
"convert_case",
"proc-macro2 1.0.66",
"quote 1.0.31",
"quote 1.0.32",
"rustc_version 0.4.0",
"syn 1.0.109",
]
@ -850,9 +850,9 @@ checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591"
[[package]]
name = "either"
version = "1.8.1"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
[[package]]
name = "encoding_rs"
@ -888,9 +888,9 @@ dependencies = [
[[package]]
name = "errno"
version = "0.3.1"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
dependencies = [
"errno-dragonfly",
"libc",
@ -934,7 +934,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.31",
"quote 1.0.32",
"syn 1.0.109",
"synstructure",
]
@ -952,12 +952,9 @@ dependencies = [
[[package]]
name = "fastrand"
version = "1.9.0"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be"
dependencies = [
"instant",
]
checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
[[package]]
name = "flate2"
@ -1231,7 +1228,7 @@ dependencies = [
"mac",
"markup5ever 0.11.0",
"proc-macro2 1.0.66",
"quote 1.0.31",
"quote 1.0.32",
"syn 1.0.109",
]
@ -1426,26 +1423,6 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "instant"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "io-lifetimes"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2"
dependencies = [
"hermit-abi",
"libc",
"windows-sys",
]
[[package]]
name = "iovec"
version = "0.1.4"
@ -1468,7 +1445,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
dependencies = [
"hermit-abi",
"rustix 0.38.4",
"rustix",
"windows-sys",
]
@ -1539,12 +1516,6 @@ version = "0.2.147"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
[[package]]
name = "linux-raw-sys"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
[[package]]
name = "linux-raw-sys"
version = "0.4.3"
@ -1784,9 +1755,9 @@ checksum = "ab250442c86f1850815b5d268639dff018c0627022bc1940eb2d642ca1ce12f0"
[[package]]
name = "num-traits"
version = "0.2.15"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
dependencies = [
"autocfg 1.1.0",
]
@ -1844,8 +1815,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.31",
"syn 2.0.26",
"quote 1.0.32",
"syn 2.0.27",
]
[[package]]
@ -1962,8 +1933,8 @@ dependencies = [
"pest",
"pest_meta",
"proc-macro2 1.0.66",
"quote 1.0.31",
"syn 2.0.26",
"quote 1.0.32",
"syn 2.0.27",
]
[[package]]
@ -2064,8 +2035,8 @@ dependencies = [
"phf_generator 0.11.2",
"phf_shared 0.11.2",
"proc-macro2 1.0.66",
"quote 1.0.31",
"syn 2.0.26",
"quote 1.0.32",
"syn 2.0.27",
]
[[package]]
@ -2192,9 +2163,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.31"
version = "1.0.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0"
checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965"
dependencies = [
"proc-macro2 1.0.66",
]
@ -2368,15 +2339,16 @@ dependencies = [
[[package]]
name = "redis"
version = "0.23.0"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f"
checksum = "ff5d95dd18a4d76650f0c2607ed8ebdbf63baf9cb934e1c233cd220c694db1d7"
dependencies = [
"combine",
"itoa 1.0.9",
"percent-encoding 2.3.0",
"ryu",
"sha1_smol",
"socket2",
"url 2.4.0",
]
@ -2409,9 +2381,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.3.3"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310"
checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294"
dependencies = [
"aho-corasick",
"memchr",
@ -2543,20 +2515,6 @@ dependencies = [
"semver 1.0.18",
]
[[package]]
name = "rustix"
version = "0.37.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06"
dependencies = [
"bitflags 1.3.2",
"errno",
"io-lifetimes",
"libc",
"linux-raw-sys 0.3.8",
"windows-sys",
]
[[package]]
name = "rustix"
version = "0.38.4"
@ -2566,7 +2524,7 @@ dependencies = [
"bitflags 2.3.3",
"errno",
"libc",
"linux-raw-sys 0.4.3",
"linux-raw-sys",
"windows-sys",
]
@ -2631,9 +2589,9 @@ dependencies = [
[[package]]
name = "security-framework"
version = "2.9.1"
version = "2.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8"
checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
@ -2644,9 +2602,9 @@ dependencies = [
[[package]]
name = "security-framework-sys"
version = "2.9.0"
version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7"
checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"
dependencies = [
"core-foundation-sys",
"libc",
@ -2704,29 +2662,29 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "serde"
version = "1.0.171"
version = "1.0.178"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9"
checksum = "60363bdd39a7be0266a520dab25fdc9241d2f987b08a01e01f0ec6d06a981348"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.171"
version = "1.0.178"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682"
checksum = "f28482318d6641454cb273da158647922d1be6b5a2fcc6165cd89ebdd7ed576b"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.31",
"syn 2.0.26",
"quote 1.0.32",
"syn 2.0.27",
]
[[package]]
name = "serde_json"
version = "1.0.103"
version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
dependencies = [
"itoa 1.0.9",
"ryu",
@ -2902,7 +2860,7 @@ dependencies = [
"phf_generator 0.7.24",
"phf_shared 0.7.24",
"proc-macro2 1.0.66",
"quote 1.0.31",
"quote 1.0.32",
"string_cache_shared",
]
@ -2915,7 +2873,7 @@ dependencies = [
"phf_generator 0.10.0",
"phf_shared 0.10.0",
"proc-macro2 1.0.66",
"quote 1.0.31",
"quote 1.0.32",
]
[[package]]
@ -2942,18 +2900,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.31",
"quote 1.0.32",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.26"
version = "2.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970"
checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.31",
"quote 1.0.32",
"unicode-ident",
]
@ -2964,22 +2922,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.31",
"quote 1.0.32",
"syn 1.0.109",
"unicode-xid 0.2.4",
]
[[package]]
name = "tempfile"
version = "3.6.0"
version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6"
checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998"
dependencies = [
"autocfg 1.1.0",
"cfg-if 1.0.0",
"fastrand",
"redox_syscall 0.3.5",
"rustix 0.37.23",
"rustix",
"windows-sys",
]
@ -3005,22 +2962,22 @@ dependencies = [
[[package]]
name = "thiserror"
version = "1.0.43"
version = "1.0.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42"
checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.43"
version = "1.0.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f"
checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.31",
"syn 2.0.26",
"quote 1.0.32",
"syn 2.0.27",
]
[[package]]
@ -3174,8 +3131,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.31",
"syn 2.0.26",
"quote 1.0.32",
"syn 2.0.27",
]
[[package]]
@ -3496,8 +3453,8 @@ dependencies = [
"log",
"once_cell",
"proc-macro2 1.0.66",
"quote 1.0.31",
"syn 2.0.26",
"quote 1.0.32",
"syn 2.0.27",
"wasm-bindgen-shared",
]
@ -3519,7 +3476,7 @@ version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
dependencies = [
"quote 1.0.31",
"quote 1.0.32",
"wasm-bindgen-macro-support",
]
@ -3530,8 +3487,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.31",
"syn 2.0.26",
"quote 1.0.32",
"syn 2.0.27",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@ -3554,7 +3511,7 @@ dependencies = [
[[package]]
name = "websurfx"
version = "0.14.1"
version = "0.15.0"
dependencies = [
"actix-files",
"actix-web",
@ -3717,18 +3674,18 @@ dependencies = [
[[package]]
name = "zstd"
version = "0.12.3+zstd.1.5.2"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806"
checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "6.0.5+zstd.1.5.4"
version = "6.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b"
checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581"
dependencies = [
"libc",
"zstd-sys",

View File

@ -1,6 +1,6 @@
[package]
name = "websurfx"
version = "0.14.1"
version = "0.15.0"
edition = "2021"
description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
repository = "https://github.com/neon-mmd/websurfx"

View File

@ -22,6 +22,7 @@ static CONFIG_FILE_NAME: &str = "config.lua";
/// * `logging` - It stores the option to whether enable or disable logs.
/// * `debug` - It stores the option to whether enable or disable debug mode.
/// * `upstream_search_engines` - It stores all the engine names that were enabled by the user.
/// * `request_timeout` - It stores the time (secs) which controls the server request timeout.
#[derive(Clone)]
pub struct Config {
pub port: u16,
@ -32,6 +33,7 @@ pub struct Config {
pub logging: bool,
pub debug: bool,
pub upstream_search_engines: Vec<String>,
pub request_timeout: u8,
}
/// Configuration options for the aggregator.
@ -80,6 +82,7 @@ impl Config {
.into_iter()
.filter_map(|(key, value)| value.then_some(key))
.collect(),
request_timeout: globals.get::<_, u8>("request_timeout")?,
})
})
}

View File

@ -29,6 +29,7 @@ impl SearchEngine for DuckDuckGo {
/// * `query` - Takes the user provided query to query to the upstream search engine with.
/// * `page` - Takes an u32 as an argument.
/// * `user_agent` - Takes a random user agent string as an argument.
/// * `request_timeout` - Takes a time (secs) as a value which controls the server request timeout.
///
/// # Errors
///
@ -41,6 +42,7 @@ impl SearchEngine for DuckDuckGo {
query: String,
page: u32,
user_agent: String,
request_timeout: u8,
) -> Result<HashMap<String, RawSearchResult>, EngineError> {
// Page number can be missing or empty string and so appropriate handling is required
// so that upstream server recieves valid page number.
@ -90,7 +92,7 @@ impl SearchEngine for DuckDuckGo {
);
let document: Html = Html::parse_document(
&DuckDuckGo::fetch_html_from_upstream(self, url, header_map).await?,
&DuckDuckGo::fetch_html_from_upstream(self, url, header_map, request_timeout).await?,
);
let no_result: Selector = Selector::parse(".no-results")

View File

@ -50,11 +50,12 @@ pub trait SearchEngine {
&self,
url: String,
header_map: reqwest::header::HeaderMap,
request_timeout: u8,
) -> Result<String, EngineError> {
// fetch the html from upstream search engine
Ok(reqwest::Client::new()
.get(url)
.timeout(Duration::from_secs(30)) // Add timeout to request to avoid DDOSing the server
.timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
.headers(header_map) // add spoofed headers to emulate human behaviour
.send()
.await
@ -71,5 +72,6 @@ pub trait SearchEngine {
query: String,
page: u32,
user_agent: String,
request_timeout: u8,
) -> Result<HashMap<String, RawSearchResult>, EngineError>;
}

View File

@ -27,6 +27,7 @@ impl SearchEngine for Searx {
/// * `query` - Takes the user provided query to query to the upstream search engine with.
/// * `page` - Takes an u32 as an argument.
/// * `user_agent` - Takes a random user agent string as an argument.
/// * `request_timeout` - Takes a time (secs) as a value which controls the server request timeout.
///
/// # Errors
///
@ -40,6 +41,7 @@ impl SearchEngine for Searx {
query: String,
page: u32,
user_agent: String,
request_timeout: u8,
) -> Result<HashMap<String, RawSearchResult>, EngineError> {
// Page number can be missing or empty string and so appropriate handling is required
// so that upstream server recieves valid page number.
@ -70,8 +72,9 @@ impl SearchEngine for Searx {
);
header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse().into_report().change_context(EngineError::UnexpectedError)?);
let document: Html =
Html::parse_document(&Searx::fetch_html_from_upstream(self, url, header_map).await?);
let document: Html = Html::parse_document(
&Searx::fetch_html_from_upstream(self, url, header_map, request_timeout).await?,
);
let no_result: Selector = Selector::parse("#urls>.dialog-error>p")
.map_err(|_| Report::new(EngineError::UnexpectedError))

View File

@ -51,6 +51,7 @@ type FutureVec = Vec<JoinHandle<Result<HashMap<String, RawSearchResult>, Report<
/// * `random_delay` - Accepts a boolean value to add a random delay before making the request.
/// * `debug` - Accepts a boolean value to enable or disable debug mode option.
/// * `upstream_search_engines` - Accepts a vector of search engine names which was selected by the
/// * `request_timeout` - Accepts a time (secs) as a value which controls the server request timeout.
/// user through the UI or the config file.
///
/// # Error
@ -64,6 +65,7 @@ pub async fn aggregate(
random_delay: bool,
debug: bool,
upstream_search_engines: Vec<String>,
request_timeout: u8,
) -> Result<SearchResults, Box<dyn std::error::Error>> {
let user_agent: String = random_user_agent();
let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
@ -92,9 +94,11 @@ pub async fn aggregate(
.map(|search_engine| {
let query: String = query.clone();
let user_agent: String = user_agent.clone();
tokio::spawn(
async move { search_engine.results(query, page, user_agent.clone()).await },
)
tokio::spawn(async move {
search_engine
.results(query, page, user_agent.clone(), request_timeout)
.await
})
})
.collect();

View File

@ -146,6 +146,7 @@ async fn results(
config.aggregator.random_delay,
config.debug,
cookie_value.engines,
config.request_timeout,
)
.await?
}
@ -156,6 +157,7 @@ async fn results(
config.aggregator.random_delay,
config.debug,
config.upstream_search_engines.clone(),
config.request_timeout,
)
.await?
}

View File

@ -8,6 +8,7 @@ binding_ip = "127.0.0.1" --ip address on the which server should be launched.
production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users (more than one))
-- if production_use is set to true
-- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.
request_timeout = 30 -- timeout for the search requests sent to the upstream search engines to be fetched (value in seconds).
-- ### Website ###
-- The different colorschemes provided are: