✨ feat: config option to configure the number of connections to have in the reqwest pool (#552)

* 🔧 Config: Updated `number_of_https_connections` for the HTTPS connection pool.

* (feat) : added the max idle pool per host option

* 📝 docs(config): add documentation back to the `http_adaptive_window_size` option (#532)

---------

Co-authored-by: neon_arch <mustafadhuleb53@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

src/config/parser.rs

@@ -48,6 +48,8 @@ pub struct Config {
     pub tcp_connection_keep_alive: u8,
     /// It stores the pool idle connection timeout in seconds.
     pub pool_idle_connection_timeout: u8,
+    /// It stores the number of https connections to keep in the pool.
+    pub number_of_https_connections: u8,
 }
 
 impl Config {
@@ -139,6 +141,7 @@ impl Config {
             request_timeout: globals.get::<_, u8>("request_timeout")?,
             tcp_connection_keep_alive: globals.get::<_, u8>("tcp_connection_keep_alive")?,
             pool_idle_connection_timeout: globals.get::<_, u8>("pool_idle_connection_timeout")?,
+            number_of_https_connections: globals.get::<_, u8>("number_of_https_connections")?,
             threads,
             client_connection_keep_alive: globals.get::<_, u8>("client_connection_keep_alive")?,
             rate_limiter: RateLimiter {

src/models/aggregation_models.rs

@@ -11,7 +11,9 @@ use thesaurus::synonyms;
 /// A named struct to store the raw scraped search results scraped search results from the
 /// upstream search engines before aggregating it.It derives the Clone trait which is needed
 /// to write idiomatic rust using `Iterators`.
+///
 ///   (href url in html in simple words).
+///
 #[derive(Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct SearchResult {

src/results/aggregator.rs

@@ -81,6 +81,7 @@ pub async fn aggregate(
                 config.pool_idle_connection_timeout as u64,
             ))
             .tcp_keepalive(Duration::from_secs(config.tcp_connection_keep_alive as u64))
+            .pool_max_idle_per_host(config.number_of_https_connections as usize)
             .connect_timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
             .https_only(true)
             .gzip(true)

websurfx/config.lua

@@ -18,6 +18,8 @@ rate_limiter = {
 }
 -- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles
 https_adaptive_window_size = false
+
+number_of_https_connections = 10 -- the number of https connections that should be available in the connection pool.
 -- Set keep-alive timer in seconds; keeps clients connected to the HTTP server, different from the connection to upstream search engines
 client_connection_keep_alive = 120