From 05272884bac1e6068f58180e964227126b9f9067 Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Mon, 22 May 2023 01:13:06 +0000 Subject: [PATCH 1/4] supports the option to add a random delay --- src/config_parser/parser.rs | 13 +++++++++++++ src/search_results_handler/aggregator.rs | 10 +++++++--- src/server/routes.rs | 2 +- websurfx/config.lua | 5 +++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index 4625bd8..21a9bf5 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -20,6 +20,14 @@ pub struct Config { pub binding_ip_addr: String, pub style: Style, pub redis_connection_url: String, + pub aggregator: AggreatorConfig, +} + +/// Configuration options for the aggregator. +#[derive(Clone)] +pub struct AggreatorConfig { + /// Whether to introduce a random delay before sending the request to the search engine. + pub random_delay: bool, } impl Config { @@ -41,6 +49,8 @@ impl Config { .load(&fs::read_to_string("./websurfx/config.lua")?) .exec()?; + let aggregator_config = globals.get::<_, rlua::Table>("aggregator")?; + Ok(Config { port: globals.get::<_, u16>("port")?, binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?, @@ -49,6 +59,9 @@ impl Config { globals.get::<_, String>("colorscheme")?, ), redis_connection_url: globals.get::<_, String>("redis_connection_url")?, + aggregator: AggreatorConfig { + random_delay: aggregator_config.get::<_, bool>("random_delay")?, + }, }) }) } diff --git a/src/search_results_handler/aggregator.rs b/src/search_results_handler/aggregator.rs index 5133094..8b86972 100644 --- a/src/search_results_handler/aggregator.rs +++ b/src/search_results_handler/aggregator.rs @@ -29,6 +29,7 @@ use crate::engines::{duckduckgo, searx}; /// /// * `query` - Accepts a string to query with the above upstream search engines. /// * `page` - Accepts an u32 page number. +/// * `random_delay` - Accepts a boolean value to add a random delay before making the request. /// /// # Error /// @@ -38,14 +39,17 @@ use crate::engines::{duckduckgo, searx}; pub async fn aggregate( query: &str, page: u32, + random_delay: bool, ) -> Result> { let user_agent: String = random_user_agent(); let mut result_map: HashMap = HashMap::new(); // Add a random delay before making the request. - let mut rng = rand::thread_rng(); - let delay_secs = rng.gen_range(1..10); - std::thread::sleep(Duration::from_secs(delay_secs)); + if random_delay { + let mut rng = rand::thread_rng(); + let delay_secs = rng.gen_range(1..10); + std::thread::sleep(Duration::from_secs(delay_secs)); + } // fetch results from upstream search engines simultaneously/concurrently. let (ddg_map_results, searx_map_results) = join!( diff --git a/src/server/routes.rs b/src/server/routes.rs index 85c522d..0f84cc9 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -127,7 +127,7 @@ pub async fn search( } Err(_) => { let mut results_json: crate::search_results_handler::aggregation_models::SearchResults = - aggregate(query, page).await?; + aggregate(query, page, config.aggregator.random_delay).await?; results_json.add_style(config.style.clone()); redis_cache .cache_results(serde_json::to_string(&results_json)?, &page_url)?; diff --git a/websurfx/config.lua b/websurfx/config.lua index 916a9b3..7dfd515 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -19,3 +19,8 @@ theme = "simple" -- the theme name which should be used for the website -- Caching redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. + +-- Aggregator +aggregator = { + random_delay = false, -- whether to add random delay before sending the request to the search engine +} \ No newline at end of file From 9773cee38dd312696bd17168e5758176a5563b8e Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Tue, 23 May 2023 00:56:44 +0000 Subject: [PATCH 2/4] change the 'aggregator' option to 'production_use' --- src/config_parser/parser.rs | 15 +++++++++++---- websurfx/config.lua | 7 +++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index 21a9bf5..4b73a73 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -49,7 +49,16 @@ impl Config { .load(&fs::read_to_string("./websurfx/config.lua")?) .exec()?; - let aggregator_config = globals.get::<_, rlua::Table>("aggregator")?; + let production_use = globals.get::<_, bool>("production_use")?; + let aggregator_config = if production_use { + AggreatorConfig { + random_delay: true, + } + } else { + AggreatorConfig { + random_delay: false, + } + }; Ok(Config { port: globals.get::<_, u16>("port")?, @@ -59,9 +68,7 @@ impl Config { globals.get::<_, String>("colorscheme")?, ), redis_connection_url: globals.get::<_, String>("redis_connection_url")?, - aggregator: AggreatorConfig { - random_delay: aggregator_config.get::<_, bool>("random_delay")?, - }, + aggregator: aggregator_config }) }) } diff --git a/websurfx/config.lua b/websurfx/config.lua index 7dfd515..1c0be7d 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -20,7 +20,6 @@ theme = "simple" -- the theme name which should be used for the website -- Caching redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. --- Aggregator -aggregator = { - random_delay = false, -- whether to add random delay before sending the request to the search engine -} \ No newline at end of file +production_use = false -- whether to use production mode or not +-- if production_use is set to true + -- there will be a random delay before sending the request to the search engines, this is to prevent the search engines from blocking the ip address \ No newline at end of file From 4b70a74ff6196a00d8878333ab4b56b065427230 Mon Sep 17 00:00:00 2001 From: zhou fan <1247714429@qq.com> Date: Tue, 23 May 2023 17:30:36 +0800 Subject: [PATCH 3/4] Update websurfx/config.lua Co-authored-by: neon_arch --- websurfx/config.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/websurfx/config.lua b/websurfx/config.lua index 1c0be7d..c30f376 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -20,6 +20,6 @@ theme = "simple" -- the theme name which should be used for the website -- Caching redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. -production_use = false -- whether to use production mode or not +production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) -- if production_use is set to true - -- there will be a random delay before sending the request to the search engines, this is to prevent the search engines from blocking the ip address \ No newline at end of file + -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. \ No newline at end of file From ea013e718edae23136befa9609100960431ab5f4 Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Tue, 23 May 2023 10:00:35 +0000 Subject: [PATCH 4/4] make format happy --- src/config_parser/parser.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index 4b73a73..f8dac14 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -51,9 +51,7 @@ impl Config { let production_use = globals.get::<_, bool>("production_use")?; let aggregator_config = if production_use { - AggreatorConfig { - random_delay: true, - } + AggreatorConfig { random_delay: true } } else { AggreatorConfig { random_delay: false, @@ -68,7 +66,7 @@ impl Config { globals.get::<_, String>("colorscheme")?, ), redis_connection_url: globals.get::<_, String>("redis_connection_url")?, - aggregator: aggregator_config + aggregator: aggregator_config, }) }) }