Merge branch 'rolling' into ability-to-have-config-on-diff-paths

.github/workflows/rust_format.yml

@@ -19,7 +19,16 @@ jobs:
           profile: minimal
           toolchain: stable
           components: rustfmt, clippy
-
+      - name: Format
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: -- --check
+      - name: Clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all-features --all-targets --all
       - name: Run cargo check
         uses: actions-rs/cargo@v1
         with:

public/templates/index.html

@@ -1,6 +1,6 @@
 {{>header this}}
 <main class="search-container">
-    <img src="images/fps_logo.png" alt="Websurfx meta-search engine logo" />
+    <img src="../images/websurfx_logo.png" alt="Websurfx meta-search engine logo" />
     {{>search_bar}}
 </main>
 <script src="static/index.js"></script>

src/config_parser/parser.rs

@@ -24,6 +24,14 @@ pub struct Config {
     pub binding_ip_addr: String,
     pub style: Style,
     pub redis_connection_url: String,
+    pub aggregator: AggreatorConfig,
+}
+
+/// Configuration options for the aggregator.
+#[derive(Clone)]
+pub struct AggreatorConfig {
+    /// Whether to introduce a random delay before sending the request to the search engine.
+    pub random_delay: bool,
 }
 
 impl Config {
@@ -45,6 +53,15 @@ impl Config {
                 )?)
                 .exec()?;
 
+            let production_use = globals.get::<_, bool>("production_use")?;
+            let aggregator_config = if production_use {
+                AggreatorConfig { random_delay: true }
+            } else {
+                AggreatorConfig {
+                    random_delay: false,
+                }
+            };
+
             Ok(Config {
                 port: globals.get::<_, u16>("port")?,
                 binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?,
@@ -53,6 +70,7 @@ impl Config {
                     globals.get::<_, String>("colorscheme")?,
                 ),
                 redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
+                aggregator: aggregator_config,
             })
         })
     }

src/search_results_handler/aggregator.rs

@@ -29,6 +29,7 @@ use crate::engines::{duckduckgo, searx};
 ///
 /// * `query` - Accepts a string to query with the above upstream search engines.
 /// * `page` - Accepts an u32 page number.
+/// * `random_delay` - Accepts a boolean value to add a random delay before making the request.
 ///
 /// # Error
 ///
@@ -38,14 +39,17 @@ use crate::engines::{duckduckgo, searx};
 pub async fn aggregate(
     query: &str,
     page: u32,
+    random_delay: bool,
 ) -> Result<SearchResults, Box<dyn std::error::Error>> {
     let user_agent: String = random_user_agent();
     let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
 
     // Add a random delay before making the request.
+    if random_delay {
         let mut rng = rand::thread_rng();
         let delay_secs = rng.gen_range(1..10);
         std::thread::sleep(Duration::from_secs(delay_secs));
+    }
 
     // fetch results from upstream search engines simultaneously/concurrently.
     let (ddg_map_results, searx_map_results) = join!(

src/server/routes.rs

@@ -127,7 +127,7 @@ pub async fn search(
                     }
                     Err(_) => {
                         let mut results_json: crate::search_results_handler::aggregation_models::SearchResults =
-                            aggregate(query, page).await?;
+                            aggregate(query, page, config.aggregator.random_delay).await?;
                         results_json.add_style(config.style.clone());
                         redis_cache
                             .cache_results(serde_json::to_string(&results_json)?, &page_url)?;

websurfx/config.lua

@@ -19,3 +19,7 @@ theme = "simple" -- the theme name which should be used for the website
 
 -- Caching
 redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.
+
+production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users)
+-- if production_use is set to true
+  -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.