Merge ed661174ba into bf7e73f9ff

Cargo.lock

@@ -3038,6 +3038,7 @@ dependencies = [
  "system-configuration",
  "tokio 1.36.0",
  "tokio-rustls",
+ "tokio-socks",
  "tokio-util",
  "tower-service",
  "url 2.5.0",
@@ -3675,6 +3676,26 @@ dependencies = [
  "libflate",
 ]
 
+[[package]]
+name = "thiserror"
+version = "1.0.60"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.60"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524"
+dependencies = [
+ "proc-macro2 1.0.78",
+ "quote 1.0.35",
+ "syn 2.0.52",
+]
+
 [[package]]
 name = "thousands"
 version = "0.2.0"
@@ -3879,6 +3900,18 @@ dependencies = [
  "tokio 1.36.0",
 ]
 
+[[package]]
+name = "tokio-socks"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51165dfa029d2a65969413a6cc96f354b86b464498702f174a4efa13608fd8c0"
+dependencies = [
+ "either",
+ "futures-util",
+ "thiserror",
+ "tokio 1.36.0",
+]
+
 [[package]]
 name = "tokio-sync"
 version = "0.1.8"

Cargo.toml

@@ -17,6 +17,7 @@ reqwest = { version = "0.11.24", default-features = false, features = [
     "rustls-tls",
     "brotli",
     "gzip",
+	"socks"
 ] }
 tokio = { version = "1.32.0", features = [
     "rt-multi-thread",

src/config/parser.rs

@@ -6,6 +6,7 @@ use crate::handler::{file_path, FileType};
 use crate::models::parser_models::{AggregatorConfig, RateLimiter, Style};
 use log::LevelFilter;
 use mlua::Lua;
+use reqwest::Proxy;
 use std::{collections::HashMap, fs, thread::available_parallelism};
 
 /// A named struct which stores the parsed config file options.
@@ -48,6 +49,9 @@ pub struct Config {
     pub tcp_connection_keep_alive: u8,
     /// It stores the pool idle connection timeout in seconds.
     pub pool_idle_connection_timeout: u8,
+
+    /// Url of the proxy to use for outgoing requests.
+    pub proxy: Option<Proxy>,
 }
 
 impl Config {
@@ -118,6 +122,15 @@ impl Config {
             _ => parsed_cet,
         };
 
+        let proxy_str = globals.get::<_, String>("proxy")?;
+        let proxy = match Proxy::all(proxy_str) {
+            Ok(proxy) => Some(proxy),
+            Err(_) => {
+                log::error!("Invalid proxy url, defaulting to no proxy.");
+                None
+            }
+        };
+
         Ok(Config {
             port: globals.get::<_, u16>("port")?,
             binding_ip: globals.get::<_, String>("binding_ip")?,
@@ -148,6 +161,7 @@ impl Config {
             safe_search,
             #[cfg(any(feature = "redis-cache", feature = "memory-cache"))]
             cache_expiry_time,
+            proxy,
         })
     }
 }

src/results/aggregator.rs

@@ -76,7 +76,7 @@ pub async fn aggregate(
     safe_search: u8,
 ) -> Result<SearchResults, Box<dyn std::error::Error>> {
     let client = CLIENT.get_or_init(|| {
-        ClientBuilder::new()
+        let mut cb = ClientBuilder::new()
             .timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
             .pool_idle_timeout(Duration::from_secs(
                 config.pool_idle_connection_timeout as u64,
@@ -86,9 +86,13 @@ pub async fn aggregate(
             .https_only(true)
             .gzip(true)
             .brotli(true)
-            .http2_adaptive_window(config.adaptive_window)
-            .build()
-            .unwrap()
+            .http2_adaptive_window(config.adaptive_window);
+
+        if config.proxy.is_some() {
+            cb = cb.proxy(config.proxy.clone().unwrap());
+        }
+
+        cb.build().unwrap()
     });
 
     let user_agent: &str = random_user_agent();
@@ -247,6 +251,7 @@ pub async fn filter_with_lists(
 
     Ok(())
 }
+
 /// Sorts  SearchResults by relevance score.
 /// <br> sort_unstable is used as its faster,stability is not an issue on our side.
 /// For reasons why, check out [`this`](https://rust-lang.github.io/rfcs/1884-unstable-sort.html)
@@ -262,6 +267,7 @@ fn sort_search_results(results: &mut [SearchResult]) {
             .unwrap_or(Ordering::Less)
     })
 }
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -291,8 +297,8 @@ mod tests {
                 url: "https://www.rust-lang.org/".to_owned(),
                 description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
                 engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()],
-                relevance_score:0.0
-            },)
+                relevance_score: 0.0,
+            }, )
         );
 
         // Create a temporary file with regex patterns
@@ -342,7 +348,7 @@ mod tests {
                 url: "https://www.rust-lang.org/".to_owned(),
                 description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
                 engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()],
-                relevance_score:0.0
+                relevance_score: 0.0,
             },
         ));
 

websurfx/config.lua

@@ -73,3 +73,5 @@ upstream_search_engines = {
     Mojeek = false,
     Bing = false,
 } -- select the upstream search engines from which the results should be fetched.
+
+proxy = "" -- Proxy to send outgoing requests through. Set to empty string to disable.