Merge branch 'rolling' into ability-to-have-config-on-diff-paths

.github/workflows/rust_format.yml

@@ -19,7 +19,16 @@ jobs:
           profile: minimal
           toolchain: stable
           components: rustfmt, clippy
-
+      - name: Format
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: -- --check
+      - name: Clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all-features --all-targets --all
       - name: Run cargo check
         uses: actions-rs/cargo@v1
         with:

public/templates/index.html

@@ -1,6 +1,6 @@
 {{>header this}}
 <main class="search-container">
-    <img src="images/fps_logo.png" alt="Websurfx meta-search engine logo" />
+    <img src="../images/websurfx_logo.png" alt="Websurfx meta-search engine logo" />
     {{>search_bar}}
 </main>
 <script src="static/index.js"></script>

src/cache/mod.rs

@@ -1 +1 @@
-pub mod cacher; 
+pub mod cacher;

src/config_parser/parser.rs

@@ -24,6 +24,14 @@ pub struct Config {
     pub binding_ip_addr: String,
     pub style: Style,
     pub redis_connection_url: String,
+    pub aggregator: AggreatorConfig,
+}
+
+/// Configuration options for the aggregator.
+#[derive(Clone)]
+pub struct AggreatorConfig {
+    /// Whether to introduce a random delay before sending the request to the search engine.
+    pub random_delay: bool,
 }
 
 impl Config {
@@ -45,6 +53,15 @@ impl Config {
                 )?)
                 .exec()?;
 
+            let production_use = globals.get::<_, bool>("production_use")?;
+            let aggregator_config = if production_use {
+                AggreatorConfig { random_delay: true }
+            } else {
+                AggreatorConfig {
+                    random_delay: false,
+                }
+            };
+
             Ok(Config {
                 port: globals.get::<_, u16>("port")?,
                 binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?,
@@ -53,6 +70,7 @@ impl Config {
                     globals.get::<_, String>("colorscheme")?,
                 ),
                 redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
+                aggregator: aggregator_config,
             })
         })
     }

src/search_results_handler/aggregation_models.rs

@@ -116,7 +116,7 @@ impl RawSearchResult {
     }
 }
 
-/// A named struct to store, serialize, deserialize the all the search results scraped and 
+/// A named struct to store, serialize, deserialize the all the search results scraped and
 /// aggregated from the upstream search engines.
 ///
 /// # Fields

src/search_results_handler/aggregator.rs

@@ -29,6 +29,7 @@ use crate::engines::{duckduckgo, searx};
 ///
 /// * `query` - Accepts a string to query with the above upstream search engines.
 /// * `page` - Accepts an u32 page number.
+/// * `random_delay` - Accepts a boolean value to add a random delay before making the request.
 ///
 /// # Error
 ///
@@ -38,14 +39,17 @@ use crate::engines::{duckduckgo, searx};
 pub async fn aggregate(
     query: &str,
     page: u32,
+    random_delay: bool,
 ) -> Result<SearchResults, Box<dyn std::error::Error>> {
     let user_agent: String = random_user_agent();
     let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
 
     // Add a random delay before making the request.
-    let mut rng = rand::thread_rng();
+    if random_delay {
-    let delay_secs = rng.gen_range(1..10);
+        let mut rng = rand::thread_rng();
-    std::thread::sleep(Duration::from_secs(delay_secs));
+        let delay_secs = rng.gen_range(1..10);
+        std::thread::sleep(Duration::from_secs(delay_secs));
+    }
 
     // fetch results from upstream search engines simultaneously/concurrently.
     let (ddg_map_results, searx_map_results) = join!(

src/server/routes.rs

@@ -81,10 +81,10 @@ pub async fn search(
                     .insert_header(("location", "/"))
                     .finish())
             } else {
-                let page_url: String;  // Declare the page_url variable without initializing it
+                let page_url: String; // Declare the page_url variable without initializing it
 
                 // ...
-                
+
                 let page = match params.page {
                     Some(page_number) => {
                         if page_number <= 1 {
@@ -98,7 +98,7 @@ pub async fn search(
                                 "http://{}:{}/search?q={}&page={}",
                                 config.binding_ip_addr, config.port, query, page_number
                             );
-                
+
                             page_number
                         }
                     }
@@ -110,11 +110,11 @@ pub async fn search(
                             req.uri(),
                             1
                         );
-                
+
                         1
                     }
                 };
-                                              
+
                 // fetch the cached results json.
                 let cached_results_json = redis_cache.cached_results_json(&page_url);
                 // check if fetched results was indeed fetched or it was an error and if so
@@ -127,7 +127,7 @@ pub async fn search(
                     }
                     Err(_) => {
                         let mut results_json: crate::search_results_handler::aggregation_models::SearchResults =
-                            aggregate(query, page).await?;
+                            aggregate(query, page, config.aggregator.random_delay).await?;
                         results_json.add_style(config.style.clone());
                         redis_cache
                             .cache_results(serde_json::to_string(&results_json)?, &page_url)?;

websurfx/config.lua

@@ -19,3 +19,7 @@ theme = "simple" -- the theme name which should be used for the website
 
 -- Caching
 redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.
+
+production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users)
+-- if production_use is set to true
+  -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.