Merge branch 'rolling' into FEAT/394_clear-button-for-the-search-bar

Cargo.lock

@@ -59,9 +59,9 @@ dependencies = [
 
 [[package]]
 name = "actix-governor"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2e7b88f3804e01bd4191fdb08650430bbfcb43d3d9b2890064df3551ec7d25b"
+checksum = "0954b0f27aabd8f56bb03f2a77b412ddf3f8c034a3c27b2086c1fc75415760df"
 dependencies = [
  "actix-http",
  "actix-web",
@@ -1174,9 +1174,9 @@ dependencies = [
 
 [[package]]
 name = "error-stack"
-version = "0.4.1"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27a72baa257b5e0e2de241967bc5ee8f855d6072351042688621081d66b2a76b"
+checksum = "fe413319145d1063f080f27556fd30b1d70b01e2ba10c2a6e40d4be982ffc5d1"
 dependencies = [
  "anyhow",
  "rustc_version 0.4.1",
@@ -2395,12 +2395,31 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
 
+[[package]]
+name = "num-bigint"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-conv"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
 
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -3114,9 +3133,9 @@ dependencies = [
 
 [[package]]
 name = "redis"
-version = "0.25.4"
+version = "0.27.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0d7a6955c7511f60f3ba9e86c6d02b3c3f144f8c24b288d1f4e18074ab8bbec"
+checksum = "a7e86f5670bd8b028edfb240f0616cad620705b31ec389d55e4f3da2c38dcd48"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -3125,6 +3144,7 @@ dependencies = [
  "futures 0.3.30",
  "futures-util",
  "itoa 1.0.11",
+ "num-bigint",
  "percent-encoding 2.3.1",
  "pin-project-lite",
  "ryu",

Cargo.toml

@@ -46,22 +46,22 @@ mlua = { version = "0.9.9", features = [
     "luajit",
     "vendored",
 ], default-features = false }
-redis = { version = "0.25.4", features = [
+redis = { version = "0.27.2", features = [
     "tokio-comp",
     "connection-manager",
     "tcp_nodelay"
 ], default-features = false, optional = true }
 blake3 = { version = "1.5.4", default-features = false }
-error-stack = { version = "0.4.0", default-features = false, features = [
+error-stack = { version = "0.5.0", default-features = false, features = [
     "std",
 ] }
 async-trait = { version = "0.1.80", default-features = false }
-regex = { version = "1.9.4", features = ["perf"], default-features = false }
+regex = { version = "1.11.0", features = ["perf"], default-features = false }
 futures = { version = "0.3.30", default-features = false, features = ["alloc"] }
 dhat = { version = "0.3.2", optional = true, default-features = false }
 mimalloc = { version = "0.1.43", default-features = false }
 async-once-cell = { version = "0.5.3", default-features = false }
-actix-governor = { version = "0.5.0", default-features = false }
+actix-governor = { version = "0.6.0", default-features = false }
 moka = { version = "0.12.8", optional = true, default-features = false, features = [
     "future",
 ] }
@@ -91,7 +91,7 @@ itertools = {version = "0.13.0", default-features = false}
 [dev-dependencies]
 rusty-hook = { version = "^0.11.2", default-features = false }
 criterion = { version = "0.5.1", default-features = false }
-tempfile = { version = "3.10.1", default-features = false }
+tempfile = { version = "3.13.0", default-features = false }
 
 [build-dependencies]
 lightningcss = { version = "1.0.0-alpha.57", default-features = false, features = [

src/config/parser.rs

@@ -48,6 +48,8 @@ pub struct Config {
     pub tcp_connection_keep_alive: u8,
     /// It stores the pool idle connection timeout in seconds.
     pub pool_idle_connection_timeout: u8,
+    /// It stores the number of https connections to keep in the pool.
+    pub number_of_https_connections: u8,
 }
 
 impl Config {
@@ -139,6 +141,7 @@ impl Config {
             request_timeout: globals.get::<_, u8>("request_timeout")?,
             tcp_connection_keep_alive: globals.get::<_, u8>("tcp_connection_keep_alive")?,
             pool_idle_connection_timeout: globals.get::<_, u8>("pool_idle_connection_timeout")?,
+            number_of_https_connections: globals.get::<_, u8>("number_of_https_connections")?,
             threads,
             client_connection_keep_alive: globals.get::<_, u8>("client_connection_keep_alive")?,
             rate_limiter: RateLimiter {

src/models/aggregation_models.rs

@@ -11,7 +11,9 @@ use thesaurus::synonyms;
 /// A named struct to store the raw scraped search results scraped search results from the
 /// upstream search engines before aggregating it.It derives the Clone trait which is needed
 /// to write idiomatic rust using `Iterators`.
+///
 ///   (href url in html in simple words).
+///
 #[derive(Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct SearchResult {

src/results/aggregator.rs

@@ -81,6 +81,7 @@ pub async fn aggregate(
                 config.pool_idle_connection_timeout as u64,
             ))
             .tcp_keepalive(Duration::from_secs(config.tcp_connection_keep_alive as u64))
+            .pool_max_idle_per_host(config.number_of_https_connections as usize)
             .connect_timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
             .https_only(true)
             .gzip(true)

websurfx/config.lua

@@ -18,6 +18,8 @@ rate_limiter = {
 }
 -- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles
 https_adaptive_window_size = false
+
+number_of_https_connections = 10 -- the number of https connections that should be available in the connection pool.
 -- Set keep-alive timer in seconds; keeps clients connected to the HTTP server, different from the connection to upstream search engines
 client_connection_keep_alive = 120