Merge branch 'rolling' into FEAT/427_export-import-settings-to-and-from-a-json-file-support-for-the-ui

Cargo.lock

@@ -59,9 +59,9 @@ dependencies = [
 
 [[package]]
 name = "actix-governor"
-version = "0.5.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2e7b88f3804e01bd4191fdb08650430bbfcb43d3d9b2890064df3551ec7d25b"
+checksum = "072a3d7907b945b0956f9721e01c117ad5765ce5be2fd9bb1e44a117c669de22"
 dependencies = [
  "actix-http",
  "actix-web",
@@ -1110,11 +1110,12 @@ dependencies = [
 
 [[package]]
 name = "dashmap"
-version = "5.5.3"
+version = "6.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
+checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
 dependencies = [
  "cfg-if 1.0.0",
+ "crossbeam-utils 0.8.20",
  "hashbrown 0.14.5",
  "lock_api 0.4.12",
  "once_cell",
@@ -1264,9 +1265,9 @@ dependencies = [
 
 [[package]]
 name = "error-stack"
-version = "0.4.1"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27a72baa257b5e0e2de241967bc5ee8f855d6072351042688621081d66b2a76b"
+checksum = "fe413319145d1063f080f27556fd30b1d70b01e2ba10c2a6e40d4be982ffc5d1"
 dependencies = [
  "anyhow",
  "rustc_version 0.4.1",
@@ -1568,14 +1569,15 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
 
 [[package]]
 name = "governor"
-version = "0.6.3"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68a7f542ee6b35af73b06abc0dad1c1bae89964e4e253bc4b587b91c9637867b"
+checksum = "0746aa765db78b521451ef74221663b57ba595bf83f75d0ce23cc09447c8139f"
 dependencies = [
  "cfg-if 1.0.0",
  "dashmap",
- "futures 0.3.31",
+ "futures-sink",
  "futures-timer",
+ "futures-util",
  "no-std-compat",
  "nonzero_ext",
  "parking_lot 0.12.3",
@@ -2497,12 +2499,31 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
 
+[[package]]
+name = "num-bigint"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-conv"
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
 
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -3220,9 +3241,9 @@ dependencies = [
 
 [[package]]
 name = "redis"
-version = "0.25.4"
+version = "0.27.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0d7a6955c7511f60f3ba9e86c6d02b3c3f144f8c24b288d1f4e18074ab8bbec"
+checksum = "81cccf17a692ce51b86564334614d72dcae1def0fd5ecebc9f02956da74352b5"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -3231,11 +3252,12 @@ dependencies = [
  "futures 0.3.31",
  "futures-util",
  "itoa 1.0.11",
+ "num-bigint",
  "percent-encoding 2.3.1",
  "pin-project-lite",
  "ryu",
  "tokio 1.41.0",
- "tokio-retry",
+ "tokio-retry2",
  "tokio-util",
  "url 2.5.2",
 ]
@@ -3359,6 +3381,7 @@ dependencies = [
  "sync_wrapper",
  "tokio 1.41.0",
  "tokio-rustls",
+ "tokio-socks",
  "tokio-util",
  "tower-service",
  "url 2.5.2",
@@ -4182,10 +4205,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "tokio-retry"
+name = "tokio-retry2"
-version = "0.3.0"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
+checksum = "903934dba1c4c2f2e9cb460ef10b5695e0b0ecad3bf9ee7c8675e540c5e8b2d1"
 dependencies = [
  "pin-project",
  "rand 0.8.5",
@@ -4203,6 +4226,18 @@ dependencies = [
  "tokio 1.41.0",
 ]
 
+[[package]]
+name = "tokio-socks"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f"
+dependencies = [
+ "either",
+ "futures-util",
+ "thiserror",
+ "tokio 1.41.0",
+]
+
 [[package]]
 name = "tokio-sync"
 version = "0.1.8"

Cargo.toml

@@ -17,9 +17,10 @@ reqwest = { version = "0.12.5", default-features = false, features = [
     "rustls-tls",
     "brotli",
     "gzip",
-    "http2"
+    "http2",
+    "socks",
 ] }
-tokio = { version = "1.32.0", features = [
+tokio = { version = "1.41.0", features = [
     "rt-multi-thread",
     "macros",
     "fs",
@@ -46,22 +47,22 @@ mlua = { version = "0.9.9", features = [
     "luajit",
     "vendored",
 ], default-features = false }
-redis = { version = "0.25.4", features = [
+redis = { version = "0.27.5", features = [
     "tokio-comp",
     "connection-manager",
     "tcp_nodelay"
 ], default-features = false, optional = true }
 blake3 = { version = "1.5.4", default-features = false }
-error-stack = { version = "0.4.0", default-features = false, features = [
+error-stack = { version = "0.5.0", default-features = false, features = [
     "std",
 ] }
 async-trait = { version = "0.1.80", default-features = false }
-regex = { version = "1.9.4", features = ["perf"], default-features = false }
+regex = { version = "1.11.1", features = ["perf"], default-features = false }
-futures = { version = "0.3.30", default-features = false, features = ["alloc"] }
+futures = { version = "0.3.31", default-features = false, features = ["alloc"] }
 dhat = { version = "0.3.2", optional = true, default-features = false }
 mimalloc = { version = "0.1.43", default-features = false }
 async-once-cell = { version = "0.5.3", default-features = false }
-actix-governor = { version = "0.5.0", default-features = false }
+actix-governor = { version = "0.7.0", default-features = false }
 moka = { version = "0.12.8", optional = true, default-features = false, features = [
     "future",
 ] }
@@ -95,7 +96,7 @@ actix-multipart = { version = "0.7.2", default-features = false, features = [
 [dev-dependencies]
 rusty-hook = { version = "^0.11.2", default-features = false }
 criterion = { version = "0.5.1", default-features = false }
-tempfile = { version = "3.10.1", default-features = false }
+tempfile = { version = "3.13.0", default-features = false }
 
 [build-dependencies]
 lightningcss = { version = "1.0.0-alpha.57", default-features = false, features = [

public/images/close.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-x"><line x1="18" y1="6" x2="6" y2="18"></line><line x1="6" y1="6" x2="18" y2="18"></line></svg>

public/static/index.js

@@ -1,34 +1,6 @@
 /**
- * Selects the input element for the search box
+* A function that clears the search input text when the clear button is clicked.
- * @type {HTMLInputElement}
+*/
- */
+function clearSearchText() {
-const searchBox = document.querySelector('input')
+    document.querySelector('.search_bar > input').value = ''
-
-/**
- * Redirects the user to the search results page with the query parameter
- */
-function searchWeb() {
-    const query = searchBox.value.trim()
-    try {
-        let safeSearchLevel = document.querySelector('.search_options select').value
-        if (query) {
-            window.location.href = `search?q=${encodeURIComponent(
-                query,
-            )}&safesearch=${encodeURIComponent(safeSearchLevel)}`
-        }
-    } catch (error) {
-        if (query) {
-            window.location.href = `search?q=${encodeURIComponent(query)}`
-        }
-    }
 }
-
-/**
- * Listens for the 'Enter' key press event on the search box and calls the searchWeb function
- * @param {KeyboardEvent} e - The keyboard event object
- */
-searchBox.addEventListener('keyup', (e) => {
-    if (e.key === 'Enter') {
-        searchWeb()
-    }
-})

public/static/pagination.js

@@ -1,39 +0,0 @@
-/**
- * Navigates to the next page by incrementing the current page number in the URL query string.
- * @returns {void}
- */
-function navigate_forward() {
-    let url = new URL(window.location);
-    let searchParams = url.searchParams;
-
-    let q = searchParams.get('q');
-    let page = parseInt(searchParams.get('page'));
-
-    if (isNaN(page)) {
-        page = 1;
-    } else {
-        page++;
-    }
-
-    window.location.href = `${url.origin}${url.pathname}?q=${encodeURIComponent(q)}&page=${page}`;
-}
-
-/**
- * Navigates to the previous page by decrementing the current page number in the URL query string.
- * @returns {void}
- */
-function navigate_backward() {
-    let url = new URL(window.location);
-    let searchParams = url.searchParams;
-
-    let q = searchParams.get('q');
-    let page = parseInt(searchParams.get('page'));
-
-    if (isNaN(page)) {
-        page = 0;
-    } else if (page > 0) {
-        page--;
-    }
-
-    window.location.href = `${url.origin}${url.pathname}?q=${encodeURIComponent(q)}&page=${page}`;
-}

public/static/search_area_options.js

@@ -1,18 +0,0 @@
-document.addEventListener(
-    'DOMContentLoaded',
-    () => {
-        let url = new URL(window.location)
-        let searchParams = url.searchParams
-
-        let safeSearchLevel = searchParams.get('safesearch')
-
-        if (
-            safeSearchLevel >= 0 &&
-            safeSearchLevel <= 2 &&
-            safeSearchLevel !== null
-        ) {
-            document.querySelector('.search_options select').value = safeSearchLevel
-        }
-    },
-    false,
-)

public/static/themes/simple.css

@@ -74,6 +74,11 @@ button {
   font-size: 1.6rem;
 }
 
+.search_bar input::-webkit-search-results-button,
+.search_bar input::-webkit-search-cancel-button{
+    display: none;
+}
+
 .search_bar input:focus {
   outline: 2px solid var(--foreground-color);
 }
@@ -443,7 +448,7 @@ footer div {
   align-items: center;
 }
 
-.page_navigation button {
+.page_navigation a {
   background: var(--background-color);
   color: var(--foreground-color);
   padding: 1rem;
@@ -452,7 +457,7 @@ footer div {
   border: none;
 }
 
-.page_navigation button:active {
+.page_navigation a:active {
   filter: brightness(1.2);
 }
 

src/config/parser.rs

@@ -6,6 +6,7 @@ use crate::handler::{file_path, FileType};
 use crate::models::parser_models::{AggregatorConfig, RateLimiter, Style};
 use log::LevelFilter;
 use mlua::Lua;
+use reqwest::Proxy;
 use std::{collections::HashMap, fs, thread::available_parallelism};
 
 /// A named struct which stores the parsed config file options.
@@ -48,6 +49,10 @@ pub struct Config {
     pub tcp_connection_keep_alive: u8,
     /// It stores the pool idle connection timeout in seconds.
     pub pool_idle_connection_timeout: u8,
+    /// Url of the proxy to use for outgoing requests.
+    pub proxy: Option<Proxy>,
+    /// It stores the number of https connections to keep in the pool.
+    pub number_of_https_connections: u8,
 }
 
 impl Config {
@@ -118,6 +123,14 @@ impl Config {
             _ => parsed_cet,
         };
 
+        let proxy_opt = globals.get::<_, Option<String>>("proxy")?;
+        let proxy = proxy_opt.and_then(|proxy_str| {
+            Proxy::all(proxy_str).ok().and_then(|_| {
+                log::error!("Invalid proxy url, defaulting to no proxy.");
+                None
+            })
+        });
+
         Ok(Config {
             port: globals.get::<_, u16>("port")?,
             binding_ip: globals.get::<_, String>("binding_ip")?,
@@ -139,6 +152,7 @@ impl Config {
             request_timeout: globals.get::<_, u8>("request_timeout")?,
             tcp_connection_keep_alive: globals.get::<_, u8>("tcp_connection_keep_alive")?,
             pool_idle_connection_timeout: globals.get::<_, u8>("pool_idle_connection_timeout")?,
+            number_of_https_connections: globals.get::<_, u8>("number_of_https_connections")?,
             threads,
             client_connection_keep_alive: globals.get::<_, u8>("client_connection_keep_alive")?,
             rate_limiter: RateLimiter {
@@ -148,6 +162,7 @@ impl Config {
             safe_search,
             #[cfg(any(feature = "redis-cache", feature = "memory-cache"))]
             cache_expiry_time,
+            proxy,
         })
     }
 }

src/engines/librex.rs

@@ -30,7 +30,7 @@ impl LibreX {
         Ok(Self {
             parser: SearchResultParser::new(
                 ".text-result-container>p",
-                ".text-result-container",
+                ".text-result-wrapper",
                 ".text-result-wrapper>a>h2",
                 ".text-result-wrapper>a",
                 ".text-result-wrapper>span",

src/lib.rs

@@ -91,7 +91,7 @@ pub fn run(
             .wrap(cors)
             .wrap(Governor::new(
                 &GovernorConfigBuilder::default()
-                    .per_second(config.rate_limiter.time_limit as u64)
+                    .seconds_per_request(config.rate_limiter.time_limit as u64)
                     .burst_size(config.rate_limiter.number_of_requests as u32)
                     .finish()
                     .unwrap(),

src/models/aggregation_models.rs

@@ -11,7 +11,9 @@ use thesaurus::synonyms;
 /// A named struct to store the raw scraped search results scraped search results from the
 /// upstream search engines before aggregating it.It derives the Clone trait which is needed
 /// to write idiomatic rust using `Iterators`.
+///
 ///   (href url in html in simple words).
+///
 #[derive(Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct SearchResult {

src/results/aggregator.rs

@@ -75,19 +75,24 @@ pub async fn aggregate(
     safe_search: u8,
 ) -> Result<SearchResults, Box<dyn std::error::Error>> {
     let client = CLIENT.get_or_init(|| {
-        ClientBuilder::new()
+        let mut cb = ClientBuilder::new()
             .timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
             .pool_idle_timeout(Duration::from_secs(
                 config.pool_idle_connection_timeout as u64,
             ))
             .tcp_keepalive(Duration::from_secs(config.tcp_connection_keep_alive as u64))
+            .pool_max_idle_per_host(config.number_of_https_connections as usize)
             .connect_timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
             .https_only(true)
             .gzip(true)
             .brotli(true)
-            .http2_adaptive_window(config.adaptive_window)
+            .http2_adaptive_window(config.adaptive_window);
-            .build()
+
-            .unwrap()
+        if config.proxy.is_some() {
+            cb = cb.proxy(config.proxy.clone().unwrap());
+        }
+
+        cb.build().unwrap()
     });
 
     let user_agent: &str = random_user_agent();
@@ -241,6 +246,7 @@ pub async fn filter_with_lists(
 
     Ok(())
 }
+
 /// Sorts  SearchResults by relevance score.
 /// <br> sort_unstable is used as its faster,stability is not an issue on our side.
 /// For reasons why, check out [`this`](https://rust-lang.github.io/rfcs/1884-unstable-sort.html)
@@ -256,6 +262,7 @@ fn sort_search_results(results: &mut [SearchResult]) {
             .unwrap_or(Ordering::Less)
     })
 }
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/server/routes/search.rs

@@ -83,7 +83,7 @@ pub async fn search(
             let next_page = page + 1;
 
             // Add a random delay before making the request.
-            if config.aggregator.random_delay || !config.debug {
+            if config.aggregator.random_delay || config.debug {
                 let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.subsec_nanos() as f32;
                 let delay = ((nanos / 1_0000_0000 as f32).floor() as u64) + 1;
                 tokio::time::sleep(Duration::from_secs(delay)).await;
@@ -127,6 +127,7 @@ pub async fn search(
                     &config.style.theme,
                     &config.style.animation,
                     query,
+                    page,
                     &results.0,
                 )
                 .0,

src/templates/partials/bar.rs

@@ -14,9 +14,13 @@ use maud::{html, Markup, PreEscaped};
 /// It returns the compiled html code for the search bar as a result.
 pub fn bar(query: &str) -> Markup {
     html!(
+        (PreEscaped("<form action=\"/search\">"))
         (PreEscaped("<div class=\"search_bar\">"))
-            input type="search" name="search-box" value=(query) placeholder="Type to search";
+            input type="search" name="q" value=(query) placeholder="Type to search";
-            button type="submit" onclick="searchWeb()" {
+            button type="button" onclick="clearSearchText()" {
+                img src="./images/close.svg" alt="Clear button icon for clearing search input text";
+            }
+            button type="submit" {
                 img src="./images/magnifying_glass.svg" alt="Info icon for error box";
             }
     )

src/templates/partials/search_bar.rs

@@ -29,7 +29,7 @@ pub fn search_bar(
             (bar(query))
                 .error_box {
                    @if !engine_errors_info.is_empty(){
-                      button onclick="toggleErrorBox()" class="error_box_toggle_button"{
+                      button type="button" onclick="toggleErrorBox()" class="error_box_toggle_button"{
                          img src="./images/warning.svg" alt="Info icon for error box";
                       }
                       .dropdown_error_box{
@@ -43,7 +43,7 @@ pub fn search_bar(
                       }
                    }
                    @else {
-                      button onclick="toggleErrorBox()" class="error_box_toggle_button"{
+                      button type="button" onclick="toggleErrorBox()" class="error_box_toggle_button"{
                          img src="./images/info.svg" alt="Warning icon for error box";
                       }
                       .dropdown_error_box {
@@ -56,10 +56,10 @@ pub fn search_bar(
             (PreEscaped("</div>"))
             .search_options {
                @if safe_search_level >= 3 {
-                   (PreEscaped("<select name=\"safe_search_levels\" disabled>"))
+                   (PreEscaped("<select name=\"safesearch\" disabled>"))
                }
                @else{
-                   (PreEscaped("<select name=\"safe_search_levels\">"))
+                   (PreEscaped(format!("<select name=\"safesearch\" value=\"{}\">", safe_search_level)))
                }
                @for (idx, name) in SAFE_SEARCH_LEVELS_NAME.iter().enumerate() {
                    @if (safe_search_level as usize) == idx {
@@ -71,6 +71,7 @@ pub fn search_bar(
                }
                (PreEscaped("</select>"))
             }
+            (PreEscaped("</form>"))
         }
     )
 }

src/templates/views/search.rs

@@ -24,6 +24,7 @@ pub fn search(
     theme: &str,
     animation: &Option<String>,
     query: &str,
+    page: u32,
     search_results: &SearchResults,
 ) -> Markup {
     html!(
@@ -108,15 +109,14 @@ pub fn search(
               }
             }
             .page_navigation {
-               button type="button" onclick="navigate_backward()"{
+               a href=(format!("/search?q={}&safesearch={}&page={}", query, search_results.safe_search_level, if page > 1 {page-1} else {1})) {
                    (PreEscaped("&#8592;")) "previous"
                }
-               button type="button" onclick="navigate_forward()"{"next" (PreEscaped("&#8594;"))}
+               a href=(format!("/search?q={}&safesearch={}&page={}", query, search_results.safe_search_level, page+2)) {
+                  "next" (PreEscaped("&#8594;"))}
             }
         }
         script src="static/index.js"{}
-        script src="static/search_area_options.js"{}
-        script src="static/pagination.js"{}
         script src="static/error_box.js"{}
         (footer())
     )

websurfx/config.lua

@@ -18,6 +18,8 @@ rate_limiter = {
 }
 -- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles
 https_adaptive_window_size = false
+
+number_of_https_connections = 10 -- the number of https connections that should be available in the connection pool.
 -- Set keep-alive timer in seconds; keeps clients connected to the HTTP server, different from the connection to upstream search engines
 client_connection_keep_alive = 120
 
@@ -73,3 +75,5 @@ upstream_search_engines = {
     Mojeek = false,
     Bing = false,
 } -- select the upstream search engines from which the results should be fetched.
+
+proxy = nil -- Proxy to send outgoing requests through. Set to nil to disable.