diff --git a/Cargo.lock b/Cargo.lock index de9a820..fa9e96c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,9 +59,9 @@ dependencies = [ [[package]] name = "actix-governor" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2e7b88f3804e01bd4191fdb08650430bbfcb43d3d9b2890064df3551ec7d25b" +checksum = "0954b0f27aabd8f56bb03f2a77b412ddf3f8c034a3c27b2086c1fc75415760df" dependencies = [ "actix-http", "actix-web", @@ -1180,9 +1180,9 @@ dependencies = [ [[package]] name = "error-stack" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27a72baa257b5e0e2de241967bc5ee8f855d6072351042688621081d66b2a76b" +checksum = "fe413319145d1063f080f27556fd30b1d70b01e2ba10c2a6e40d4be982ffc5d1" dependencies = [ "anyhow", "rustc_version 0.4.1", @@ -2411,12 +2411,31 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3127,9 +3146,9 @@ dependencies = [ [[package]] name = "redis" -version = "0.25.4" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d7a6955c7511f60f3ba9e86c6d02b3c3f144f8c24b288d1f4e18074ab8bbec" +checksum = "a7e86f5670bd8b028edfb240f0616cad620705b31ec389d55e4f3da2c38dcd48" dependencies = [ "arc-swap", "async-trait", @@ -3138,6 +3157,7 @@ dependencies = [ "futures 0.3.30", "futures-util", "itoa 1.0.11", + "num-bigint", "percent-encoding 2.3.1", "pin-project-lite", "ryu", @@ -3164,9 +3184,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick 1.1.3", "memchr", @@ -3176,9 +3196,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick 1.1.3", "memchr", @@ -3193,9 +3213,9 @@ checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" @@ -3335,9 +3355,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.36" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno", @@ -3831,9 +3851,9 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if 1.0.0", "fastrand", diff --git a/Cargo.toml b/Cargo.toml index 2421d50..b0b8980 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,22 +47,22 @@ mlua = { version = "0.9.9", features = [ "luajit", "vendored", ], default-features = false } -redis = { version = "0.25.4", features = [ +redis = { version = "0.27.2", features = [ "tokio-comp", "connection-manager", "tcp_nodelay" ], default-features = false, optional = true } blake3 = { version = "1.5.4", default-features = false } -error-stack = { version = "0.4.0", default-features = false, features = [ +error-stack = { version = "0.5.0", default-features = false, features = [ "std", ] } async-trait = { version = "0.1.80", default-features = false } -regex = { version = "1.9.4", features = ["perf"], default-features = false } +regex = { version = "1.11.0", features = ["perf"], default-features = false } futures = { version = "0.3.30", default-features = false, features = ["alloc"] } dhat = { version = "0.3.2", optional = true, default-features = false } mimalloc = { version = "0.1.43", default-features = false } async-once-cell = { version = "0.5.3", default-features = false } -actix-governor = { version = "0.5.0", default-features = false } +actix-governor = { version = "0.6.0", default-features = false } moka = { version = "0.12.8", optional = true, default-features = false, features = [ "future", ] } @@ -92,7 +92,7 @@ itertools = {version = "0.13.0", default-features = false} [dev-dependencies] rusty-hook = { version = "^0.11.2", default-features = false } criterion = { version = "0.5.1", default-features = false } -tempfile = { version = "3.10.1", default-features = false } +tempfile = { version = "3.13.0", default-features = false } [build-dependencies] lightningcss = { version = "1.0.0-alpha.57", default-features = false, features = [ diff --git a/src/cache/cacher.rs b/src/cache/cacher.rs index 604d527..5eac473 100644 --- a/src/cache/cacher.rs +++ b/src/cache/cacher.rs @@ -546,7 +546,7 @@ impl SharedCache { /// # Arguments /// /// * `url` - It takes the search url as an argument which will be used as the key to fetch the - /// cached results from the cache. + /// cached results from the cache. /// /// # Error /// @@ -563,9 +563,9 @@ impl SharedCache { /// # Arguments /// /// * `search_results` - It takes the `SearchResults` as an argument which are results that - /// needs to be cached. + /// needs to be cached. /// * `url` - It takes the search url as an argument which will be used as the key for storing - /// results in the cache. + /// results in the cache. /// /// # Error /// diff --git a/src/config/parser.rs b/src/config/parser.rs index 8bed460..5ff9444 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -48,6 +48,8 @@ pub struct Config { pub tcp_connection_keep_alive: u8, /// It stores the pool idle connection timeout in seconds. pub pool_idle_connection_timeout: u8, + /// It stores the number of https connections to keep in the pool. + pub number_of_https_connections: u8, } impl Config { @@ -57,7 +59,7 @@ impl Config { /// # Arguments /// /// * `logging_initialized` - It takes a boolean which ensures that the logging doesn't get - /// initialized twice. Pass false if the logger has not yet been initialized. + /// initialized twice. Pass false if the logger has not yet been initialized. /// /// # Error /// @@ -139,6 +141,7 @@ impl Config { request_timeout: globals.get::<_, u8>("request_timeout")?, tcp_connection_keep_alive: globals.get::<_, u8>("tcp_connection_keep_alive")?, pool_idle_connection_timeout: globals.get::<_, u8>("pool_idle_connection_timeout")?, + number_of_https_connections: globals.get::<_, u8>("number_of_https_connections")?, threads, client_connection_keep_alive: globals.get::<_, u8>("client_connection_keep_alive")?, rate_limiter: RateLimiter { diff --git a/src/models/aggregation_models.rs b/src/models/aggregation_models.rs index 01c67a6..53aa25b 100644 --- a/src/models/aggregation_models.rs +++ b/src/models/aggregation_models.rs @@ -11,7 +11,9 @@ use thesaurus::synonyms; /// A named struct to store the raw scraped search results scraped search results from the /// upstream search engines before aggregating it.It derives the Clone trait which is needed /// to write idiomatic rust using `Iterators`. -/// (href url in html in simple words). +/// +/// (href url in html in simple words). +/// #[derive(Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SearchResult { @@ -34,7 +36,7 @@ impl SearchResult { /// /// * `title` - The title of the search result. /// * `url` - The url which is accessed when clicked on it - /// (href url in html in simple words). + /// (href url in html in simple words). /// * `description` - The description of the search result. /// * `engine` - The names of the upstream engines from which this results were provided. pub fn new(title: &str, url: &str, description: &str, engine: &[&str]) -> Self { @@ -124,7 +126,7 @@ impl EngineErrorInfo { /// # Arguments /// /// * `error` - It takes the error type which occured while fetching the result from a particular - /// search engine. + /// search engine. /// * `engine` - It takes the name of the engine that failed to provide the requested search results. pub fn new(error: &EngineError, engine: &str) -> Self { Self { @@ -177,11 +179,11 @@ impl SearchResults { /// # Arguments /// /// * `results` - Takes an argument of individual serializable `SearchResult` struct - /// and stores it into a vector of `SearchResult` structs. + /// and stores it into a vector of `SearchResult` structs. /// * `page_query` - Takes an argument of current page`s search query `q` provided in - /// the search url. + /// the search url. /// * `engine_errors_info` - Takes an array of structs which contains information regarding - /// which engines failed with their names, reason and their severity color name. + /// which engines failed with their names, reason and their severity color name. pub fn new(results: Box<[SearchResult]>, engine_errors_info: Box<[EngineErrorInfo]>) -> Self { Self { results, diff --git a/src/models/parser_models.rs b/src/models/parser_models.rs index 24b3fd8..5f578a5 100644 --- a/src/models/parser_models.rs +++ b/src/models/parser_models.rs @@ -29,7 +29,7 @@ impl Style { /// /// * `theme` - It takes the parsed theme option used to set a theme for the website. /// * `colorscheme` - It takes the parsed colorscheme option used to set a colorscheme - /// for the theme being used. + /// for the theme being used. pub fn new(theme: String, colorscheme: String, animation: Option) -> Self { Style { theme, diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index 009111b..65d5a5e 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -60,7 +60,7 @@ type FutureVec = /// * `debug` - Accepts a boolean value to enable or disable debug mode option. /// * `upstream_search_engines` - Accepts a vector of search engine names which was selected by the /// * `request_timeout` - Accepts a time (secs) as a value which controls the server request timeout. -/// user through the UI or the config file. +/// user through the UI or the config file. /// /// # Error /// @@ -81,6 +81,7 @@ pub async fn aggregate( config.pool_idle_connection_timeout as u64, )) .tcp_keepalive(Duration::from_secs(config.tcp_connection_keep_alive as u64)) + .pool_max_idle_per_host(config.number_of_https_connections as usize) .connect_timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server .https_only(true) .gzip(true) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index d43ddec..9fa6877 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -146,7 +146,7 @@ pub async fn search( /// # Arguments /// /// * `url` - It takes the url of the current page that requested the search results for a -/// particular search query. +/// particular search query. /// * `config` - It takes a parsed config struct. /// * `query` - It takes the page number as u32 value. /// * `req` - It takes the `HttpRequest` struct as a value. diff --git a/src/templates/partials/search_bar.rs b/src/templates/partials/search_bar.rs index c40244c..9c6ea9c 100644 --- a/src/templates/partials/search_bar.rs +++ b/src/templates/partials/search_bar.rs @@ -12,7 +12,7 @@ const SAFE_SEARCH_LEVELS_NAME: [&str; 3] = ["None", "Low", "Moderate"]; /// # Arguments /// /// * `engine_errors_info` - It takes the engine errors list containing errors for each upstream -/// search engine which failed to provide results as an argument. +/// search engine which failed to provide results as an argument. /// * `safe_search_level` - It takes the safe search level with values from 0-2 as an argument. /// * `query` - It takes the current search query provided by user as an argument. /// diff --git a/src/templates/partials/settings_tabs/engines.rs b/src/templates/partials/settings_tabs/engines.rs index de67ac7..ad67f80 100644 --- a/src/templates/partials/settings_tabs/engines.rs +++ b/src/templates/partials/settings_tabs/engines.rs @@ -9,7 +9,7 @@ use maud::{html, Markup}; /// # Arguments /// /// * `engine_names` - It takes the key value pair list of all available engine names and there corresponding -/// selected (enabled/disabled) value as an argument. +/// selected (enabled/disabled) value as an argument. /// /// # Returns /// diff --git a/src/templates/partials/settings_tabs/user_interface.rs b/src/templates/partials/settings_tabs/user_interface.rs index 8f685be..87b0fb3 100644 --- a/src/templates/partials/settings_tabs/user_interface.rs +++ b/src/templates/partials/settings_tabs/user_interface.rs @@ -11,9 +11,9 @@ use std::fs::read_dir; /// # Arguments /// /// * `style_type` - It takes the style type of the values `theme` and `colorscheme` as an -/// argument. +/// argument. /// * `selected_style` - It takes the currently selected style value provided via the config file -/// as an argument. +/// as an argument. /// /// # Error /// diff --git a/websurfx/config.lua b/websurfx/config.lua index a6ccf67..c8d49ac 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -18,6 +18,8 @@ rate_limiter = { } -- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles https_adaptive_window_size = false + +number_of_https_connections = 10 -- the number of https connections that should be available in the connection pool. -- Set keep-alive timer in seconds; keeps clients connected to the HTTP server, different from the connection to upstream search engines client_connection_keep_alive = 120