From d8bd2feb6ecd565ff617f17bfb90ce443e5d4295 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 2 Aug 2023 20:05:39 +0300 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20new=20config=20op?= =?UTF-8?q?tion=20to=20manage=20threads=20and=20improve=20logging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/websurfx.rs | 18 ++++++++++-------- src/config/parser.rs | 39 ++++++++++++++++++++++++++++++++++++--- src/lib.rs | 3 +++ websurfx/config.lua | 1 + 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index d8d7f5c..57682d9 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -4,7 +4,6 @@ //! stdout and handles the command line arguments provided and launches the `websurfx` server. use std::net::TcpListener; - use websurfx::{config::parser::Config, run}; /// The function that launches the main server and registers all the routes of the website. @@ -18,13 +17,16 @@ async fn main() -> std::io::Result<()> { // Initialize the parsed config file. let config = Config::parse().unwrap(); - // Initializing logging middleware with level set to default or info. - if config.logging || config.debug { - use env_logger::Env; - env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); - } - - log::info!("started server on port {}", config.port); + log::info!( + "started server on port {} and IP {}", + config.port, + config.binding_ip + ); + log::info!( + "Open http://{}:{}/ in your browser", + config.port, + config.binding_ip + ); let listener = TcpListener::bind((config.binding_ip.clone(), config.port))?; diff --git a/src/config/parser.rs b/src/config/parser.rs index a665246..81fa6b8 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -2,8 +2,9 @@ //! into rust readable form. use super::parser_models::Style; +use log::LevelFilter; use rlua::Lua; -use std::{collections::HashMap, format, fs, path::Path}; +use std::{collections::HashMap, format, fs, io::Write, path::Path, thread::available_parallelism}; // ------- Constants -------- static COMMON_DIRECTORY_NAME: &str = "websurfx"; @@ -23,6 +24,7 @@ static CONFIG_FILE_NAME: &str = "config.lua"; /// * `debug` - It stores the option to whether enable or disable debug mode. /// * `upstream_search_engines` - It stores all the engine names that were enabled by the user. /// * `request_timeout` - It stores the time (secs) which controls the server request timeout. +/// * `threads` - It stores the number of threads which controls the app will use to run. #[derive(Clone)] pub struct Config { pub port: u16, @@ -34,6 +36,7 @@ pub struct Config { pub debug: bool, pub upstream_search_engines: Vec, pub request_timeout: u8, + pub threads: u8, } /// Configuration options for the aggregator. @@ -64,6 +67,35 @@ impl Config { .load(&fs::read_to_string(Config::config_path()?)?) .exec()?; + let parsed_threads: u8 = globals.get::<_, u8>("threads")?; + + let debug: bool = globals.get::<_, bool>("debug")?; + let logging:bool= globals.get::<_, bool>("logging")?; + + // Initializing logging middleware with level set to default or info. + let mut log_level: LevelFilter = LevelFilter::Off; + if logging && debug == false { + log_level = LevelFilter::Info; + } else if debug { + log_level = LevelFilter::Trace; + }; + env_logger::Builder::new().filter(None, log_level).init(); + + let threads: u8 = if parsed_threads == 0 { + let total_num_of_threads:usize = available_parallelism()?.get() /2; + if debug || logging { + log::error!("Config Error: The value of `threads` option should be a non zero positive integer"); + log::info!("Falling back to using {} threads", total_num_of_threads) + } else { + std::io::stdout() + .lock() + .write_all(&format!("Config Error: The value of `threads` option should be a non zero positive integer\nFalling back to using {} threads\n", total_num_of_threads).into_bytes())?; + }; + total_num_of_threads as u8 + } else { + parsed_threads + }; + Ok(Config { port: globals.get::<_, u16>("port")?, binding_ip: globals.get::<_, String>("binding_ip")?, @@ -75,14 +107,15 @@ impl Config { aggregator: AggregatorConfig { random_delay: globals.get::<_, bool>("production_use")?, }, - logging: globals.get::<_, bool>("logging")?, - debug: globals.get::<_, bool>("debug")?, + logging, + debug, upstream_search_engines: globals .get::<_, HashMap>("upstream_search_engines")? .into_iter() .filter_map(|(key, value)| value.then_some(key)) .collect(), request_timeout: globals.get::<_, u8>("request_timeout")?, + threads, }) }) } diff --git a/src/lib.rs b/src/lib.rs index e226e14..ea81746 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,6 +49,8 @@ pub fn run(listener: TcpListener, config: Config) -> std::io::Result { let handlebars_ref: web::Data = web::Data::new(handlebars); + let cloned_config_threads_opt: u8 = config.threads; + let server = HttpServer::new(move || { App::new() .app_data(handlebars_ref.clone()) @@ -70,6 +72,7 @@ pub fn run(listener: TcpListener, config: Config) -> std::io::Result { .service(routes::settings) // settings page .default_service(web::route().to(routes::not_found)) // error page }) + .workers(cloned_config_threads_opt as usize) // Start server on 127.0.0.1 with the user provided port number. for example 127.0.0.1:8080. .listen(listener)? .run(); diff --git a/websurfx/config.lua b/websurfx/config.lua index 73bffe7..4f2633c 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -1,6 +1,7 @@ -- ### General ### logging = true -- an option to enable or disable logs. debug = false -- an option to enable or disable debug mode. +threads = 10 -- the amount of threads that the app will use to run (the value should be greater than 0). -- ### Server ### port = "8080" -- port on which server should be launched From 11bcf9c98abb87258181e61894b06f8baa47c28c Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 2 Aug 2023 20:07:29 +0300 Subject: [PATCH 2/5] =?UTF-8?q?=E2=9C=A8=20feat:=20remove=20space=20from?= =?UTF-8?q?=20the=20end=20of=20the=20line?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/parser.rs b/src/config/parser.rs index 81fa6b8..079084e 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -91,7 +91,7 @@ impl Config { .lock() .write_all(&format!("Config Error: The value of `threads` option should be a non zero positive integer\nFalling back to using {} threads\n", total_num_of_threads).into_bytes())?; }; - total_num_of_threads as u8 + total_num_of_threads as u8 } else { parsed_threads }; From 26b59078b86cd813b90fb194fa5d12694b489f71 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 2 Aug 2023 20:09:07 +0300 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=9A=80=20bump=20the=20app=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 27 ++++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a0fed7f..98a6247 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -460,11 +460,12 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "6c6b2562119bf28c3439f7f02db99faf0aa1a8cdfe5772a2ee155d32227239f0" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -1912,9 +1913,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d2d1d55045829d65aad9d389139882ad623b33b904e7c9f1b10c5b8927298e5" +checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" dependencies = [ "thiserror", "ucd-trie", @@ -1922,9 +1923,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f94bca7e7a599d89dea5dfa309e217e7906c3c007fb9c3299c40b10d6a315d3" +checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" dependencies = [ "pest", "pest_generator", @@ -1932,9 +1933,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d490fe7e8556575ff6911e45567ab95e71617f43781e5c05490dc8d75c965c" +checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" dependencies = [ "pest", "pest_meta", @@ -1945,9 +1946,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2674c66ebb4b4d9036012091b537aae5878970d6999f81a265034d85b136b341" +checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" dependencies = [ "once_cell", "pest", @@ -2523,9 +2524,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.4" +version = "0.38.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +checksum = "1ee020b1716f0a80e2ace9b03441a749e402e86712f15f16fe8a8f75afac732f" dependencies = [ "bitflags 2.3.3", "errno", @@ -3518,7 +3519,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.15.3" +version = "0.16.0" dependencies = [ "actix-files", "actix-web", diff --git a/Cargo.toml b/Cargo.toml index c856b7d..ec8d159 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.15.3" +version = "0.16.0" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" From 4b4dc28cd238230e48074a1a3b94f71c0c116d78 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Thu, 3 Aug 2023 00:12:09 +0300 Subject: [PATCH 4/5] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20control=20check?= =?UTF-8?q?=20to=20avoid=20reinitializing=20logging=20&=20fix=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/websurfx.rs | 2 +- src/config/parser.rs | 26 +++++++++++++++++--------- tests/index.rs | 4 ++-- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index 57682d9..b4b989e 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -15,7 +15,7 @@ use websurfx::{config::parser::Config, run}; #[actix_web::main] async fn main() -> std::io::Result<()> { // Initialize the parsed config file. - let config = Config::parse().unwrap(); + let config = Config::parse(true).unwrap(); log::info!( "started server on port {} and IP {}", diff --git a/src/config/parser.rs b/src/config/parser.rs index 079084e..51a83ea 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -54,12 +54,17 @@ impl Config { /// A function which parses the config.lua file and puts all the parsed options in the newly /// constructed Config struct and returns it. /// + /// # Arguments + /// + /// * `logging_initialized` - It takes a boolean which ensures that the logging doesn't get + /// initialized twice. + /// /// # Error /// /// Returns a lua parse error if parsing of the config.lua file fails or has a syntax error /// or io error if the config.lua file doesn't exists otherwise it returns a newly constructed /// Config struct with all the parsed config options from the parsed config file. - pub fn parse() -> Result> { + pub fn parse(logging_initialized: bool) -> Result> { Lua::new().context(|context| -> Result> { let globals = context.globals(); @@ -72,14 +77,17 @@ impl Config { let debug: bool = globals.get::<_, bool>("debug")?; let logging:bool= globals.get::<_, bool>("logging")?; - // Initializing logging middleware with level set to default or info. - let mut log_level: LevelFilter = LevelFilter::Off; - if logging && debug == false { - log_level = LevelFilter::Info; - } else if debug { - log_level = LevelFilter::Trace; - }; - env_logger::Builder::new().filter(None, log_level).init(); + // Check whether logging has not been initialized before. + if logging_initialized { + // Initializing logging middleware with level set to default or info. + let mut log_level: LevelFilter = LevelFilter::Off; + if logging && debug == false { + log_level = LevelFilter::Info; + } else if debug { + log_level = LevelFilter::Trace; + }; + env_logger::Builder::new().filter(None, log_level).init(); + } let threads: u8 = if parsed_threads == 0 { let total_num_of_threads:usize = available_parallelism()?.get() /2; diff --git a/tests/index.rs b/tests/index.rs index 657a466..d886e13 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -8,7 +8,7 @@ fn spawn_app() -> String { // Binding to port 0 will trigger the OS to assign a port for us. let listener = TcpListener::bind("127.0.0.1:0").expect("Failed to bind random port"); let port = listener.local_addr().unwrap().port(); - let config = Config::parse().unwrap(); + let config = Config::parse(true).unwrap(); let server = run(listener, config).expect("Failed to bind address"); tokio::spawn(server); @@ -36,7 +36,7 @@ async fn test_index() { assert_eq!(res.status(), 200); let handlebars = handlebars(); - let config = Config::parse().unwrap(); + let config = Config::parse(false).unwrap(); let template = handlebars.render("index", &config.style).unwrap(); assert_eq!(res.text().await.unwrap(), template); } From e02fafd69c91dd7dcd3d4b00c3317632a3a6d4b7 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Thu, 3 Aug 2023 01:03:56 +0300 Subject: [PATCH 5/5] =?UTF-8?q?=E2=9C=A8=20feat:=20fix=20doc=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index ea81746..48a2340 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,7 +34,7 @@ use handler::public_paths::public_path; /// use std::net::TcpListener; /// use websurfx::{config::parser::Config, run}; /// -/// let config = Config::parse().unwrap(); +/// let config = Config::parse(true).unwrap(); /// let listener = TcpListener::bind("127.0.0.1:8080").expect("Failed to bind address"); /// let server = run(listener,config).expect("Failed to start server"); /// ```