From bfeb81270ca0da99ca02608fdecf593c585a7819 Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Sat, 30 Dec 2023 19:19:22 -0700 Subject: [PATCH 01/37] fix: modify cache key with engines used for search --- src/server/routes/search.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index 9261051..fe07935 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -123,11 +123,24 @@ async fn results( config.safe_search, ); - let cache_key = format!( + let mut cache_key = format!( "http://{}:{}/search?q={}&page={}&safesearch={}", config.binding_ip, config.port, query, page, safe_search_level ); + // Modify the cache key adding each enabled search engine to the string + if let Some(cookie_value) = &cookie_value { + let mut engines: Vec = cookie_value + .engines + .iter() + .map(|s| String::from(*s)) + .collect::>(); + + // We sort the list of engine so the cache keys will match between users. The cookie's list of engines is unordered. + engines.sort(); + cache_key = cache_key + &(engines.join("")); + } + // fetch the cached results json. let cached_results = cache.cached_results(&cache_key).await; // check if fetched cache results was indeed fetched or it was an error and if so From 286bcf1bd359c827e5c7139eab2320a6c4bcc2bf Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Sat, 30 Dec 2023 22:12:43 -0700 Subject: [PATCH 02/37] fix: added a config parse backup --- src/server/routes/search.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index fe07935..10f6982 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -139,6 +139,16 @@ async fn results( // We sort the list of engine so the cache keys will match between users. The cookie's list of engines is unordered. engines.sort(); cache_key = cache_key + &(engines.join("")); + } else { + let mut engines: Vec = config + .upstream_search_engines + .iter() + .filter(|map| *map.1) + .map(|map| String::from(&(*map.0))) + .collect(); + + engines.sort(); + cache_key = cache_key + &(engines.join("")); } // fetch the cached results json. From 4f4cb220ccb4706a8d60767f561954b5ede50d29 Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Sat, 30 Dec 2023 22:18:15 -0700 Subject: [PATCH 03/37] fix: changed dereferencing --- src/server/routes/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index 10f6982..25ddc9a 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -144,7 +144,7 @@ async fn results( .upstream_search_engines .iter() .filter(|map| *map.1) - .map(|map| String::from(&(*map.0))) + .map(|map| String::from(map.0)) .collect(); engines.sort(); From db1115f19edaf165cb1b813fc90a99599dec33b0 Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Mon, 1 Jan 2024 19:36:18 -0700 Subject: [PATCH 04/37] fix: add comma separator and format macro --- src/server/routes/search.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index 25ddc9a..35bd9ad 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -138,7 +138,7 @@ async fn results( // We sort the list of engine so the cache keys will match between users. The cookie's list of engines is unordered. engines.sort(); - cache_key = cache_key + &(engines.join("")); + cache_key = format!("{}{}", cache_key, engines.join(",")); } else { let mut engines: Vec = config .upstream_search_engines @@ -148,7 +148,7 @@ async fn results( .collect(); engines.sort(); - cache_key = cache_key + &(engines.join("")); + cache_key = format!("{}{}", cache_key, engines.join(",")); } // fetch the cached results json. From 86b0d3d6c9ca708aaaf6b0417d93188286daff62 Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Tue, 2 Jan 2024 16:26:04 -0700 Subject: [PATCH 05/37] Moved parsing of cookie_value, config to cache_key --- src/server/routes/search.rs | 66 ++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index 908875a..e8e0153 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -6,7 +6,7 @@ use crate::{ handler::{file_path, FileType}, models::{ aggregation_models::SearchResults, - engine_models::{EngineError, EngineHandler}, + engine_models::EngineHandler, server_models::{Cookie, SearchParams}, }, results::aggregator::aggregate, @@ -126,27 +126,27 @@ async fn results( config.binding_ip, config.port, query, page, safe_search_level ); + let mut cookie_engines: Vec = vec![]; + let mut config_engines: Vec = config + .upstream_search_engines + .iter() + .filter_map(|(engine, enabled)| enabled.then_some(engine.clone())) + .collect(); + config_engines.sort(); + // Modify the cache key adding each enabled search engine to the string if let Some(cookie_value) = &cookie_value { - let mut engines: Vec = cookie_value + cookie_engines = cookie_value .engines .iter() .map(|s| String::from(*s)) .collect::>(); // We sort the list of engine so the cache keys will match between users. The cookie's list of engines is unordered. - engines.sort(); - cache_key = format!("{}{}", cache_key, engines.join(",")); + cookie_engines.sort(); + cache_key = format!("{}{}", cache_key, cookie_engines.join(",")); } else { - let mut engines: Vec = config - .upstream_search_engines - .iter() - .filter(|map| *map.1) - .map(|map| String::from(map.0)) - .collect(); - - engines.sort(); - cache_key = format!("{}{}", cache_key, engines.join(",")); + cache_key = format!("{}{}", cache_key, config_engines.join(",")); } // fetch the cached results json. @@ -175,9 +175,10 @@ async fn results( // parse the non-empty cookie and grab the user selected engines from the // UI and use that. let mut results: SearchResults = match cookie_value { - Some(cookie_value) => { - let engines: Vec = cookie_value - .engines + // If the cookie was used before + Some(_) => { + // Use the cookie_engines Strings from before to create the EngineHandlers + let engines: Vec = cookie_engines .iter() .filter_map(|name| EngineHandler::new(name).ok()) .collect(); @@ -202,23 +203,22 @@ async fn results( } } } - None => aggregate( - query, - page, - config.aggregator.random_delay, - config.debug, - &config - .upstream_search_engines - .clone() - .into_iter() - .filter_map(|(key, value)| value.then_some(key)) - .map(|engine| EngineHandler::new(&engine)) - .collect::, error_stack::Report>>( - )?, - config.request_timeout, - safe_search_level, - ) - .await?, + // Otherwise, use the config_engines to create the EngineHandlers + None => { + aggregate( + query, + page, + config.aggregator.random_delay, + config.debug, + &config_engines + .into_iter() + .filter_map(|engine| EngineHandler::new(&engine).ok()) + .collect::>(), + config.request_timeout, + safe_search_level, + ) + .await? + } }; if results.engine_errors_info().is_empty() && results.results().is_empty() From dde117e7e67c019fee8a301be6ed455bf65038a8 Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Thu, 4 Jan 2024 11:05:09 -0700 Subject: [PATCH 06/37] modify: changed cache_key format data --- src/server/routes/search.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index e8e0153..3f880f5 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -144,9 +144,9 @@ async fn results( // We sort the list of engine so the cache keys will match between users. The cookie's list of engines is unordered. cookie_engines.sort(); - cache_key = format!("{}{}", cache_key, cookie_engines.join(",")); + cache_key = format!("{cache_key}&engines={}", cookie_engines.join(",")); } else { - cache_key = format!("{}{}", cache_key, config_engines.join(",")); + cache_key = format!("{cache_key}&engines={}", config_engines.join(",")); } // fetch the cached results json. From 7d762b372609b4afe19b6c9b8484726858f2244c Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Sun, 7 Jan 2024 16:29:39 -0700 Subject: [PATCH 07/37] refactor: moved settings parsing out of results --- src/models/server_models.rs | 23 +++++- src/server/routes/search.rs | 150 ++++++++++++------------------------ 2 files changed, 68 insertions(+), 105 deletions(-) diff --git a/src/models/server_models.rs b/src/models/server_models.rs index 4772b98..a13a7dd 100644 --- a/src/models/server_models.rs +++ b/src/models/server_models.rs @@ -2,6 +2,8 @@ //! engine website. use serde::Deserialize; +use super::parser_models::Style; + /// A named struct which deserializes all the user provided search parameters and stores them. #[derive(Deserialize)] pub struct SearchParams { @@ -19,13 +21,26 @@ pub struct SearchParams { /// A named struct which is used to deserialize the cookies fetched from the client side. #[allow(dead_code)] #[derive(Deserialize)] -pub struct Cookie<'a> { +pub struct Cookie { /// It stores the theme name used in the website. - pub theme: &'a str, + pub theme: String, /// It stores the colorscheme name used for the website theme. - pub colorscheme: &'a str, + pub colorscheme: String, /// It stores the user selected upstream search engines selected from the UI. - pub engines: Vec<&'a str>, + pub engines: Vec, /// It stores the user selected safe search level from the UI. pub safe_search_level: u8, } + +impl Cookie { + /// server_models::Cookie contructor function + pub fn build(style: &Style, mut engines: Vec, safe_search_level: u8) -> Self { + engines.sort(); + Self { + theme: style.theme.clone(), + colorscheme: style.colorscheme.clone(), + engines, + safe_search_level, + } + } +} diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index 3f880f5..edf8e1b 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -7,7 +7,7 @@ use crate::{ models::{ aggregation_models::SearchResults, engine_models::EngineHandler, - server_models::{Cookie, SearchParams}, + server_models::{self, SearchParams}, }, results::aggregator::aggregate, }; @@ -48,17 +48,37 @@ pub async fn search( .finish()); } - let get_results = |page| { - results( - &config, - &cache, - query, - page, - req.clone(), - ¶ms.safesearch, + // Closure to build a server_models::Cookie capturing local references + let build_cookie = || { + server_models::Cookie::build( + &config.style, + config + .upstream_search_engines + .clone() + .into_iter() + .filter_map(|engine_map| engine_map.1.then_some(engine_map.0)) + .collect(), + config.safe_search, ) }; + // Get search settings using the user's cookie or from the server's config + let search_settings: server_models::Cookie = match req.cookie("appCookie") { + Some(cookie_value) => { + if let Ok(cookie) = serde_json::from_str(cookie_value.value()) { + cookie + // If there's an issue parsing the cookie's value, default to the config + } else { + build_cookie() + } + } + // If there's no cookie saved, use the server's config + None => build_cookie(), + }; + + // Closure wrapping the results function capturing local references + let get_results = |page| results(&config, &cache, query, page, &search_settings); + // .max(1) makes sure that the page >= 0. let page = params.page.unwrap_or(1).max(1) - 1; @@ -105,50 +125,21 @@ async fn results( cache: &web::Data, query: &str, page: u32, - req: HttpRequest, - safe_search: &Option, + user_settings: &server_models::Cookie, ) -> Result> { // eagerly parse cookie value to evaluate safe search level - let cookie_value = req.cookie("appCookie"); + let safe_search_level = user_settings.safe_search_level; - let cookie_value: Option> = cookie_value - .as_ref() - .and_then(|cv| serde_json::from_str(cv.name_value().1).ok()); - - let safe_search_level = get_safesearch_level( - safe_search, - &cookie_value.as_ref().map(|cv| cv.safe_search_level), - config.safe_search, + let cache_key = format!( + "http://{}:{}/search?q={}&page={}&safesearch={}&engines={}", + config.binding_ip, + config.port, + query, + page, + safe_search_level, + user_settings.engines.join(",") ); - let mut cache_key = format!( - "http://{}:{}/search?q={}&page={}&safesearch={}", - config.binding_ip, config.port, query, page, safe_search_level - ); - - let mut cookie_engines: Vec = vec![]; - let mut config_engines: Vec = config - .upstream_search_engines - .iter() - .filter_map(|(engine, enabled)| enabled.then_some(engine.clone())) - .collect(); - config_engines.sort(); - - // Modify the cache key adding each enabled search engine to the string - if let Some(cookie_value) = &cookie_value { - cookie_engines = cookie_value - .engines - .iter() - .map(|s| String::from(*s)) - .collect::>(); - - // We sort the list of engine so the cache keys will match between users. The cookie's list of engines is unordered. - cookie_engines.sort(); - cache_key = format!("{cache_key}&engines={}", cookie_engines.join(",")); - } else { - cache_key = format!("{cache_key}&engines={}", config_engines.join(",")); - } - // fetch the cached results json. let cached_results = cache.cached_results(&cache_key).await; // check if fetched cache results was indeed fetched or it was an error and if so @@ -174,43 +165,16 @@ async fn results( // default selected upstream search engines from the config file otherwise // parse the non-empty cookie and grab the user selected engines from the // UI and use that. - let mut results: SearchResults = match cookie_value { - // If the cookie was used before - Some(_) => { - // Use the cookie_engines Strings from before to create the EngineHandlers - let engines: Vec = cookie_engines - .iter() - .filter_map(|name| EngineHandler::new(name).ok()) - .collect(); - - match engines.is_empty() { - false => { - aggregate( - query, - page, - config.aggregator.random_delay, - config.debug, - &engines, - config.request_timeout, - safe_search_level, - ) - .await? - } - true => { - let mut search_results = SearchResults::default(); - search_results.set_no_engines_selected(); - search_results - } - } - } - // Otherwise, use the config_engines to create the EngineHandlers - None => { + let mut results: SearchResults = match user_settings.engines.is_empty() { + false => { aggregate( query, page, config.aggregator.random_delay, config.debug, - &config_engines + &user_settings + .engines + .clone() .into_iter() .filter_map(|engine| EngineHandler::new(&engine).ok()) .collect::>(), @@ -219,6 +183,11 @@ async fn results( ) .await? } + true => { + let mut search_results = SearchResults::default(); + search_results.set_no_engines_selected(); + search_results + } }; if results.engine_errors_info().is_empty() && results.results().is_empty() @@ -259,24 +228,3 @@ fn is_match_from_filter_list( Ok(false) } - -/// A helper function which returns the safe search level based on the url params -/// and cookie value. -/// -/// # Argurments -/// -/// * `safe_search` - Safe search level from the url. -/// * `cookie` - User's cookie -/// * `default` - Safe search level to fall back to -fn get_safesearch_level(safe_search: &Option, cookie: &Option, default: u8) -> u8 { - match safe_search { - Some(ss) => { - if *ss >= 3 { - default - } else { - *ss - } - } - None => cookie.unwrap_or(default), - } -} From 6e9250c03a1de25568ed37992a624eb8769fe20c Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Mon, 8 Jan 2024 09:19:36 -0700 Subject: [PATCH 08/37] fix: safe search url parameter ignored --- src/server/routes/search.rs | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index edf8e1b..4e3d309 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -63,19 +63,21 @@ pub async fn search( }; // Get search settings using the user's cookie or from the server's config - let search_settings: server_models::Cookie = match req.cookie("appCookie") { + let mut search_settings: server_models::Cookie = match req.cookie("appCookie") { Some(cookie_value) => { - if let Ok(cookie) = serde_json::from_str(cookie_value.value()) { - cookie - // If there's an issue parsing the cookie's value, default to the config - } else { - build_cookie() + match serde_json::from_str(cookie_value.value()) { + Ok(cookie) => cookie, + // If there's an issue parsing the cookie's value, default to the config + Err(_) => build_cookie(), } } // If there's no cookie saved, use the server's config None => build_cookie(), }; + search_settings.safe_search_level = + get_safesearch_level(¶ms.safesearch, config.safe_search); + // Closure wrapping the results function capturing local references let get_results = |page| results(&config, &cache, query, page, &search_settings); @@ -228,3 +230,24 @@ fn is_match_from_filter_list( Ok(false) } + +/// A helper function which returns the safe search level based on the url params +/// and cookie value. +/// +/// # Argurments +/// +/// * `safe_search` - Safe search level from the url. +/// * `cookie` - User's cookie +/// * `default` - Safe search level to fall back to +fn get_safesearch_level(safe_search: &Option, default: u8) -> u8 { + match safe_search { + Some(ss) => { + if *ss >= 3 { + default + } else { + *ss + } + } + None => default, + } +} From d912bff94e912d77d7bfbdcf327f7adff514c45a Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Mon, 8 Jan 2024 11:46:21 -0700 Subject: [PATCH 09/37] change: revert Cookie Strings back to &str --- src/models/server_models.rs | 14 +++++++------- src/server/routes/search.rs | 16 +++++++++------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/models/server_models.rs b/src/models/server_models.rs index a13a7dd..8b19463 100644 --- a/src/models/server_models.rs +++ b/src/models/server_models.rs @@ -21,24 +21,24 @@ pub struct SearchParams { /// A named struct which is used to deserialize the cookies fetched from the client side. #[allow(dead_code)] #[derive(Deserialize)] -pub struct Cookie { +pub struct Cookie<'a> { /// It stores the theme name used in the website. - pub theme: String, + pub theme: &'a str, /// It stores the colorscheme name used for the website theme. - pub colorscheme: String, + pub colorscheme: &'a str, /// It stores the user selected upstream search engines selected from the UI. pub engines: Vec, /// It stores the user selected safe search level from the UI. pub safe_search_level: u8, } -impl Cookie { +impl<'a> Cookie<'a> { /// server_models::Cookie contructor function - pub fn build(style: &Style, mut engines: Vec, safe_search_level: u8) -> Self { + pub fn build(style: &'a Style, mut engines: Vec, safe_search_level: u8) -> Self { engines.sort(); Self { - theme: style.theme.clone(), - colorscheme: style.colorscheme.clone(), + theme: &style.theme, + colorscheme: &style.colorscheme, engines, safe_search_level, } diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index 4e3d309..cc16486 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -62,9 +62,11 @@ pub async fn search( ) }; + let cookie = req.cookie("appCookie"); + // Get search settings using the user's cookie or from the server's config - let mut search_settings: server_models::Cookie = match req.cookie("appCookie") { - Some(cookie_value) => { + let mut search_settings: server_models::Cookie<'_> = match cookie { + Some(ref cookie_value) => { match serde_json::from_str(cookie_value.value()) { Ok(cookie) => cookie, // If there's an issue parsing the cookie's value, default to the config @@ -127,10 +129,10 @@ async fn results( cache: &web::Data, query: &str, page: u32, - user_settings: &server_models::Cookie, + search_settings: &server_models::Cookie<'_>, ) -> Result> { // eagerly parse cookie value to evaluate safe search level - let safe_search_level = user_settings.safe_search_level; + let safe_search_level = search_settings.safe_search_level; let cache_key = format!( "http://{}:{}/search?q={}&page={}&safesearch={}&engines={}", @@ -139,7 +141,7 @@ async fn results( query, page, safe_search_level, - user_settings.engines.join(",") + search_settings.engines.join(",") ); // fetch the cached results json. @@ -167,14 +169,14 @@ async fn results( // default selected upstream search engines from the config file otherwise // parse the non-empty cookie and grab the user selected engines from the // UI and use that. - let mut results: SearchResults = match user_settings.engines.is_empty() { + let mut results: SearchResults = match search_settings.engines.is_empty() { false => { aggregate( query, page, config.aggregator.random_delay, config.debug, - &user_settings + &search_settings .engines .clone() .into_iter() From 80e950de3bc091e25e2e3a66a7967b48770a24e3 Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Wed, 10 Jan 2024 14:14:14 -0700 Subject: [PATCH 10/37] refactor: changed Cookie to use Cow to facilitate using references when building --- src/models/server_models.rs | 16 ++++---- src/server/routes/search.rs | 77 ++++++++++++++++--------------------- 2 files changed, 43 insertions(+), 50 deletions(-) diff --git a/src/models/server_models.rs b/src/models/server_models.rs index 8b19463..eca252e 100644 --- a/src/models/server_models.rs +++ b/src/models/server_models.rs @@ -1,5 +1,7 @@ //! This module provides the models to parse cookies and search parameters from the search //! engine website. +use std::borrow::Cow; + use serde::Deserialize; use super::parser_models::Style; @@ -23,23 +25,23 @@ pub struct SearchParams { #[derive(Deserialize)] pub struct Cookie<'a> { /// It stores the theme name used in the website. - pub theme: &'a str, + pub theme: Cow<'a, str>, /// It stores the colorscheme name used for the website theme. - pub colorscheme: &'a str, + pub colorscheme: Cow<'a, str>, /// It stores the user selected upstream search engines selected from the UI. - pub engines: Vec, + pub engines: Cow<'a, Vec>>, /// It stores the user selected safe search level from the UI. pub safe_search_level: u8, } impl<'a> Cookie<'a> { /// server_models::Cookie contructor function - pub fn build(style: &'a Style, mut engines: Vec, safe_search_level: u8) -> Self { + pub fn build(style: &'a Style, mut engines: Vec>, safe_search_level: u8) -> Self { engines.sort(); Self { - theme: &style.theme, - colorscheme: &style.colorscheme, - engines, + theme: Cow::Borrowed(&style.theme), + colorscheme: Cow::Borrowed(&style.colorscheme), + engines: Cow::Owned(engines), safe_search_level, } } diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index cc16486..9d66452 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -14,6 +14,7 @@ use crate::{ use actix_web::{get, http::header::ContentType, web, HttpRequest, HttpResponse}; use regex::Regex; use std::{ + borrow::Cow, fs::File, io::{BufRead, BufReader, Read}, }; @@ -48,37 +49,30 @@ pub async fn search( .finish()); } - // Closure to build a server_models::Cookie capturing local references - let build_cookie = || { - server_models::Cookie::build( - &config.style, - config - .upstream_search_engines - .clone() - .into_iter() - .filter_map(|engine_map| engine_map.1.then_some(engine_map.0)) - .collect(), - config.safe_search, - ) - }; - let cookie = req.cookie("appCookie"); // Get search settings using the user's cookie or from the server's config - let mut search_settings: server_models::Cookie<'_> = match cookie { - Some(ref cookie_value) => { - match serde_json::from_str(cookie_value.value()) { - Ok(cookie) => cookie, - // If there's an issue parsing the cookie's value, default to the config - Err(_) => build_cookie(), - } - } - // If there's no cookie saved, use the server's config - None => build_cookie(), - }; + let mut search_settings: server_models::Cookie<'_> = cookie + .and_then(|cookie_value| serde_json::from_str(cookie_value.value()).ok()) + .unwrap_or_else(|| { + server_models::Cookie::build( + &config.style, + config + .upstream_search_engines + .iter() + .filter_map(|(engine, enabled)| { + enabled.then_some(Cow::Borrowed(engine.as_str())) + }) + .collect(), + config.safe_search, + ) + }); - search_settings.safe_search_level = - get_safesearch_level(¶ms.safesearch, config.safe_search); + get_safesearch_level( + &mut search_settings.safe_search_level, + ¶ms.safesearch, + config.safe_search, + ); // Closure wrapping the results function capturing local references let get_results = |page| results(&config, &cache, query, page, &search_settings); @@ -178,8 +172,7 @@ async fn results( config.debug, &search_settings .engines - .clone() - .into_iter() + .iter() .filter_map(|engine| EngineHandler::new(&engine).ok()) .collect::>(), config.request_timeout, @@ -233,23 +226,21 @@ fn is_match_from_filter_list( Ok(false) } -/// A helper function which returns the safe search level based on the url params -/// and cookie value. +/// A helper function to modify the safe search level based on the url params. +/// The `safe_search` is the one in the user's cookie or +/// the default set by the server config if the cookie was missing. /// /// # Argurments /// -/// * `safe_search` - Safe search level from the url. -/// * `cookie` - User's cookie -/// * `default` - Safe search level to fall back to -fn get_safesearch_level(safe_search: &Option, default: u8) -> u8 { - match safe_search { - Some(ss) => { - if *ss >= 3 { - default - } else { - *ss - } +/// * `url_level` - Safe search level from the url. +/// * `safe_search` - User's cookie, or the safe search level set by the server +/// * `config_level` - Safe search level to fall back to +fn get_safesearch_level(safe_search: &mut u8, url_level: &Option, config_level: u8) { + if let Some(search_level) = url_level { + if *search_level >= 3 { + *safe_search = config_level + } else { + *safe_search = *search_level; } - None => default, } } From ca96a76958b71ee100894f6e570e8af7d191eef0 Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Thu, 11 Jan 2024 11:58:09 -0700 Subject: [PATCH 11/37] change: renamed get_safesearch_level variables --- src/server/routes/search.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index 9d66452..c176e65 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -236,11 +236,11 @@ fn is_match_from_filter_list( /// * `safe_search` - User's cookie, or the safe search level set by the server /// * `config_level` - Safe search level to fall back to fn get_safesearch_level(safe_search: &mut u8, url_level: &Option, config_level: u8) { - if let Some(search_level) = url_level { - if *search_level >= 3 { + if let Some(url_level) = url_level { + if *url_level >= 3 { *safe_search = config_level } else { - *safe_search = *search_level; + *safe_search = *url_level; } } } From 31c9c676eed2a85515ec353df022f6befb038b5a Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Fri, 12 Jan 2024 09:51:31 -0700 Subject: [PATCH 12/37] Change get_safe_search_level logic back --- src/server/routes/search.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index c176e65..a1c2cf3 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -68,8 +68,8 @@ pub async fn search( ) }); - get_safesearch_level( - &mut search_settings.safe_search_level, + search_settings.safe_search_level = get_safesearch_level( + &Some(search_settings.safe_search_level), ¶ms.safesearch, config.safe_search, ); @@ -235,12 +235,15 @@ fn is_match_from_filter_list( /// * `url_level` - Safe search level from the url. /// * `safe_search` - User's cookie, or the safe search level set by the server /// * `config_level` - Safe search level to fall back to -fn get_safesearch_level(safe_search: &mut u8, url_level: &Option, config_level: u8) { - if let Some(url_level) = url_level { - if *url_level >= 3 { - *safe_search = config_level - } else { - *safe_search = *url_level; +fn get_safesearch_level(cookie_level: &Option, url_level: &Option, config_level: u8) -> u8 { + match url_level { + Some(url_level) => { + if *url_level >= 3 { + config_level + } else { + *url_level + } } + None => cookie_level.unwrap_or(config_level), } } From c25cd9c3fee931df64e66d1ced10ed00fd699a12 Mon Sep 17 00:00:00 2001 From: TheNortheWind <143673137+TheNortheWind@users.noreply.github.com> Date: Tue, 16 Jan 2024 00:21:12 -0800 Subject: [PATCH 13/37] :memo: note Highlighting in the `readme` file (#488) Changed the Note highlighting from: **Note** to [!Note] Affected lines: 144, 169, 221 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index eb80fc7..7da058a 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ redis-server --port 8082 & Once you have started the server, open your preferred web browser and navigate to to start using Websurfx. -> **Note** +> [!Note] > > 1. The project is no longer in the testing phase and is now ready for production use. > 2. There are many features still missing, like `support for image search`, `different categories`, `quick apps`, etc., but they will be added soon as part of future releases. @@ -166,7 +166,7 @@ Websurfx comes loaded with several themes and color schemes, which you can apply # Multi-Language Support 🌍 -> **Note** +> [!Note] > Currently, we do not support other languages, but we will start accepting contributions regarding language support in the future. We believe language should never be a barrier to entry. **[⬆️ Back to Top](#--)** @@ -218,7 +218,7 @@ Several areas that we need a bit of help with at the moment are: # Documentation πŸ“˜ -> **Note** +> [!Note] > We welcome any contributions to the [documentation](docs) as this will benefit everyone who uses this project. **[⬆️ Back to Top](#--)** From 99f0cf111382c67c112ead70314bf75164e0e97d Mon Sep 17 00:00:00 2001 From: Siddharth Tiwari Date: Sun, 21 Jan 2024 12:22:15 +0530 Subject: [PATCH 14/37] fixed issue #490 --- .github/workflows/rust_format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust_format.yml b/.github/workflows/rust_format.yml index 1b7c9fa..14b38ce 100644 --- a/.github/workflows/rust_format.yml +++ b/.github/workflows/rust_format.yml @@ -33,7 +33,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: clippy - args: --all-targets --all + args: --all-targets --all-features --all - name: Run cargo check uses: actions-rs/cargo@v1 with: From b2971c182911fb31153ad1c955742aac0733bb97 Mon Sep 17 00:00:00 2001 From: alamin655 Date: Sun, 21 Jan 2024 15:56:16 +0530 Subject: [PATCH 15/37] =?UTF-8?q?=F0=9F=94=96=20chore(release):=20bump=20t?= =?UTF-8?q?he=20app=20version=20(#491)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1045c6d..aff946b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "1.9.0" +version = "1.9.3" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" From 0f717cc976b7ed6600d0541e7264e0b96a925047 Mon Sep 17 00:00:00 2001 From: alamin655 Date: Sun, 21 Jan 2024 15:56:41 +0530 Subject: [PATCH 16/37] =?UTF-8?q?=F0=9F=94=96=20chore(release):=20bump=20t?= =?UTF-8?q?he=20app=20version=20(#491)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 7befc09..7a41139 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4146,7 +4146,7 @@ checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" [[package]] name = "websurfx" -version = "1.9.0" +version = "1.9.3" dependencies = [ "actix-cors", "actix-files", From c762f9cf8ef21d37cca0cbeda4e40c473571c681 Mon Sep 17 00:00:00 2001 From: Spencerjibz Date: Mon, 22 Jan 2024 19:20:16 +0000 Subject: [PATCH 17/37] use redis pipeline to set multiple values at once --- src/cache/redis_cacher.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/cache/redis_cacher.rs b/src/cache/redis_cacher.rs index cfb2a2e..c775963 100644 --- a/src/cache/redis_cacher.rs +++ b/src/cache/redis_cacher.rs @@ -118,14 +118,18 @@ impl RedisCache { /// on a failure. pub async fn cache_json( &mut self, - json_results: &str, - key: &str, + json_results: impl Iterator, + keys: impl Iterator, ) -> Result<(), Report> { self.current_connection = Default::default(); + let mut pipeline = redis::Pipeline::with_capacity(3); - let mut result: Result<(), RedisError> = self.connection_pool - [self.current_connection as usize] - .set_ex(key, json_results, self.cache_ttl.into()) + for (key, json_result) in keys.zip(json_results) { + pipeline.set_ex(key, json_result, self.cache_ttl.into()); + } + + let mut result: Result<(), RedisError> = pipeline + .query_async(&mut self.connection_pool[self.current_connection as usize]) .await; // Code to check whether the current connection being used is dropped with connection error @@ -145,8 +149,10 @@ impl RedisCache { CacheError::PoolExhaustionWithConnectionDropError, )); } - result = self.connection_pool[self.current_connection as usize] - .set_ex(key, json_results, 60) + result = pipeline + .query_async( + &mut self.connection_pool[self.current_connection as usize], + ) .await; continue; } From f5cf5f9151daa2593946689d248af233175f7476 Mon Sep 17 00:00:00 2001 From: Spencerjibz Date: Mon, 22 Jan 2024 19:26:35 +0000 Subject: [PATCH 18/37] changed cache_results method to support multiple values --- src/cache/cacher.rs | 66 ++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/src/cache/cacher.rs b/src/cache/cacher.rs index d1144e3..7fa881f 100644 --- a/src/cache/cacher.rs +++ b/src/cache/cacher.rs @@ -4,6 +4,7 @@ use error_stack::Report; #[cfg(feature = "memory-cache")] use mini_moka::sync::Cache as MokaCache; +use mini_moka::sync::ConcurrentCacheExt; #[cfg(feature = "memory-cache")] use std::time::Duration; @@ -61,8 +62,8 @@ pub trait Cacher: Send + Sync { /// failure. async fn cache_results( &mut self, - search_results: &SearchResults, - url: &str, + search_results: &[SearchResults], + urls: &[String], ) -> Result<(), Report>; /// A helper function which computes the hash of the url and formats and returns it as string. @@ -332,14 +333,29 @@ impl Cacher for RedisCache { async fn cache_results( &mut self, - search_results: &SearchResults, - url: &str, + search_results: &[SearchResults], + urls: &[String], ) -> Result<(), Report> { use base64::Engine; - let bytes = self.pre_process_search_results(search_results)?; - let base64_string = base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes); - let hashed_url_string = self.hash_url(url); - self.cache_json(&base64_string, &hashed_url_string).await + let mut bytes = Vec::with_capacity(3); + + for result in search_results { + let processed = self.pre_process_search_results(result)?; + bytes.push(processed); + } + + let base64_strings = bytes + .iter() + .map(|bytes_vec| base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes_vec)); + + let mut hashed_url_strings = Vec::with_capacity(3); + + for url in urls { + let hash = self.hash_url(url); + hashed_url_strings.push(hash); + } + self.cache_json(base64_strings, hashed_url_strings.into_iter()) + .await } } /// TryInto implementation for SearchResults from Vec @@ -391,12 +407,16 @@ impl Cacher for InMemoryCache { async fn cache_results( &mut self, - search_results: &SearchResults, - url: &str, + search_results: &[SearchResults], + urls: &[String], ) -> Result<(), Report> { - let hashed_url_string = self.hash_url(url); - let bytes = self.pre_process_search_results(search_results)?; - self.cache.insert(hashed_url_string, bytes); + for (url, search_result) in urls.iter().zip(search_results.iter()) { + let hashed_url_string = self.hash_url(url); + let bytes = self.pre_process_search_results(search_result)?; + self.cache.insert(hashed_url_string, bytes); + } + + self.cache.sync(); Ok(()) } } @@ -434,11 +454,13 @@ impl Cacher for HybridCache { async fn cache_results( &mut self, - search_results: &SearchResults, - url: &str, + search_results: &[SearchResults], + urls: &[String], ) -> Result<(), Report> { - self.redis_cache.cache_results(search_results, url).await?; - self.memory_cache.cache_results(search_results, url).await?; + self.redis_cache.cache_results(search_results, urls).await?; + self.memory_cache + .cache_results(search_results, urls) + .await?; Ok(()) } @@ -460,8 +482,8 @@ impl Cacher for DisabledCache { async fn cache_results( &mut self, - _search_results: &SearchResults, - _url: &str, + _search_results: &[SearchResults], + _urls: &[String], ) -> Result<(), Report> { Ok(()) } @@ -519,11 +541,11 @@ impl SharedCache { /// on a failure. pub async fn cache_results( &self, - search_results: &SearchResults, - url: &str, + search_results: &[SearchResults], + urls: &[String], ) -> Result<(), Report> { let mut mut_cache = self.cache.lock().await; - mut_cache.cache_results(search_results, url).await + mut_cache.cache_results(search_results, urls).await } } From 6b9469e4b360e37455babc111911abe8dcfd7f7d Mon Sep 17 00:00:00 2001 From: Spencerjibz Date: Mon, 22 Jan 2024 19:29:08 +0000 Subject: [PATCH 19/37] cache the next, current and previous results in a separate task --- src/server/routes/search.rs | 69 ++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index a1c2cf3..16cfa28 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -40,6 +40,7 @@ pub async fn search( config: web::Data, cache: web::Data, ) -> Result> { + use std::sync::Arc; let params = web::Query::::from_query(req.query_string())?; match ¶ms.q { Some(query) => { @@ -79,12 +80,50 @@ pub async fn search( // .max(1) makes sure that the page >= 0. let page = params.page.unwrap_or(1).max(1) - 1; + let previous_page = page.saturating_sub(1); + let next_page = page + 1; - let (_, results, _) = join!( - get_results(page.saturating_sub(1)), - get_results(page), - get_results(page + 1) - ); + let mut results = Arc::new((SearchResults::default(), String::default())); + if page != previous_page { + let (previous_results, current_results, next_results) = join!( + get_results(previous_page), + get_results(page), + get_results(next_page) + ); + let (parsed_previous_results, parsed_next_results) = + (previous_results?, next_results?); + + let (cache_keys, results_list) = ( + [ + parsed_previous_results.1, + results.1.clone(), + parsed_next_results.1, + ], + [ + parsed_previous_results.0, + results.0.clone(), + parsed_next_results.0, + ], + ); + + results = Arc::new(current_results?); + + tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await }); + } else { + let (current_results, next_results) = + join!(get_results(page), get_results(page + 1)); + + let parsed_next_results = next_results?; + + results = Arc::new(current_results?); + + let (cache_keys, results_list) = ( + [results.1.clone(), parsed_next_results.1.clone()], + [results.0.clone(), parsed_next_results.0], + ); + + tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await }); + } Ok(HttpResponse::Ok().content_type(ContentType::html()).body( crate::templates::views::search::search( @@ -92,7 +131,7 @@ pub async fn search( &config.style.theme, &config.style.animation, query, - &results?, + &results.0, ) .0, )) @@ -124,7 +163,7 @@ async fn results( query: &str, page: u32, search_settings: &server_models::Cookie<'_>, -) -> Result> { +) -> Result<(SearchResults, String), Box> { // eagerly parse cookie value to evaluate safe search level let safe_search_level = search_settings.safe_search_level; @@ -143,7 +182,7 @@ async fn results( // check if fetched cache results was indeed fetched or it was an error and if so // handle the data accordingly. match cached_results { - Ok(results) => Ok(results), + Ok(results) => Ok((results, cache_key)), Err(_) => { if safe_search_level == 4 { let mut results: SearchResults = SearchResults::default(); @@ -153,9 +192,11 @@ async fn results( // Return early when query contains disallowed words, if flag { results.set_disallowed(); - cache.cache_results(&results, &cache_key).await?; + cache + .cache_results(&[results.clone()], &[cache_key.clone()]) + .await?; results.set_safe_search_level(safe_search_level); - return Ok(results); + return Ok((results, cache_key)); } } @@ -173,7 +214,7 @@ async fn results( &search_settings .engines .iter() - .filter_map(|engine| EngineHandler::new(&engine).ok()) + .filter_map(|engine| EngineHandler::new(engine).ok()) .collect::>(), config.request_timeout, safe_search_level, @@ -192,9 +233,11 @@ async fn results( { results.set_filtered(); } - cache.cache_results(&results, &cache_key).await?; + cache + .cache_results(&[results.clone()], &[cache_key.clone()]) + .await?; results.set_safe_search_level(safe_search_level); - Ok(results) + Ok((results, cache_key)) } } } From 29b76be45935df2729c6335da9475138d1db27fa Mon Sep 17 00:00:00 2001 From: Spencerjibz Date: Wed, 24 Jan 2024 12:47:19 +0000 Subject: [PATCH 20/37] dump up package version --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7befc09..72af913 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4146,7 +4146,7 @@ checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" [[package]] name = "websurfx" -version = "1.9.0" +version = "1.9.4" dependencies = [ "actix-cors", "actix-files", diff --git a/Cargo.toml b/Cargo.toml index 1045c6d..fdfbe32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "1.9.0" +version = "1.9.4" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" From 51214dc23ae386997ef66d5799587e598979c4b7 Mon Sep 17 00:00:00 2001 From: Spencerjibz Date: Wed, 24 Jan 2024 22:56:11 +0000 Subject: [PATCH 21/37] add support for caching N number of key-values pairs instead of only 3 --- src/cache/cacher.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/cache/cacher.rs b/src/cache/cacher.rs index 7fa881f..f323395 100644 --- a/src/cache/cacher.rs +++ b/src/cache/cacher.rs @@ -337,7 +337,11 @@ impl Cacher for RedisCache { urls: &[String], ) -> Result<(), Report> { use base64::Engine; - let mut bytes = Vec::with_capacity(3); + + // size of search_results is expected to be equal to size of urls -> key/value pairs for cache; + let search_results_len = search_results.len(); + + let mut bytes = Vec::with_capacity(search_results_len); for result in search_results { let processed = self.pre_process_search_results(result)?; @@ -348,7 +352,7 @@ impl Cacher for RedisCache { .iter() .map(|bytes_vec| base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes_vec)); - let mut hashed_url_strings = Vec::with_capacity(3); + let mut hashed_url_strings = Vec::with_capacity(search_results_len); for url in urls { let hash = self.hash_url(url); From 779908cb11c4938ad953383eab0f914b97a892e2 Mon Sep 17 00:00:00 2001 From: Yogen P Date: Sun, 28 Jan 2024 00:15:32 +0800 Subject: [PATCH 22/37] updated stargazer link (#496) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7da058a..9a7377d 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ We would like to thank the following people for their contributions and support:

- +

From fbf73634ee72059f59acb7611e07d1830247c332 Mon Sep 17 00:00:00 2001 From: abdulahad5112 <123822052+abdulahad5112@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:39:33 +0530 Subject: [PATCH 23/37] =?UTF-8?q?=F0=9F=90=9B=20Undeclared=20mini-mocha=20?= =?UTF-8?q?crate=20error=20when=20building=20the=20app=20with=20features?= =?UTF-8?q?=20other=20than=20memory-cache=20#493=20(#501)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cache/cacher.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cache/cacher.rs b/src/cache/cacher.rs index f323395..53d8050 100644 --- a/src/cache/cacher.rs +++ b/src/cache/cacher.rs @@ -4,6 +4,7 @@ use error_stack::Report; #[cfg(feature = "memory-cache")] use mini_moka::sync::Cache as MokaCache; +#[cfg(feature = "memory-cache")] use mini_moka::sync::ConcurrentCacheExt; #[cfg(feature = "memory-cache")] From 851ea314a7b9e85fa718065dd5fd0c74ddfefa55 Mon Sep 17 00:00:00 2001 From: abdulahad5112 <123822052+abdulahad5112@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:44:31 +0530 Subject: [PATCH 24/37] =?UTF-8?q?=F0=9F=90=9B=20parsed=5Fcet=20not=20found?= =?UTF-8?q?=20in=20scope=20error=20when=20building=20the=20app=20with=20th?= =?UTF-8?q?e=20no-cache=20feature=20#498=20(#502)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: alamin655 <129589283+alamin655@users.noreply.github.com> --- src/config/parser.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config/parser.rs b/src/config/parser.rs index 20a4a1a..63329c5 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -98,6 +98,7 @@ impl Config { #[cfg(any(feature = "redis-cache", feature = "memory-cache"))] let parsed_cet = globals.get::<_, u16>("cache_expiry_time")?; + #[cfg(any(feature = "redis-cache", feature = "memory-cache"))] let cache_expiry_time = match parsed_cet { 0..=59 => { log::error!( From b2cbc5eaa5dadd74e17840cbae64bb84b9feaf79 Mon Sep 17 00:00:00 2001 From: abdulahad5112 <123822052+abdulahad5112@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:50:14 +0530 Subject: [PATCH 25/37] =?UTF-8?q?=F0=9F=93=9D=20Maintained=20badge/shield?= =?UTF-8?q?=20status=20in=20the=20readme=20from=20stale=20to=20yes/maintai?= =?UTF-8?q?ned=20#500=20(#503)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: alamin655 <129589283+alamin655@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a7377d..0aeb23e 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Maintenance From 669e36591326d95aa431fe6693b3dbeead0a639a Mon Sep 17 00:00:00 2001 From: Jann Marc Villablanca <31008330+jfvillablanca@users.noreply.github.com> Date: Tue, 30 Jan 2024 21:37:50 +0800 Subject: [PATCH 26/37] feat: add new helper function to fetch upstream search engine JSON response (#504) Co-authored-by: neon_arch --- src/models/engine_models.rs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/models/engine_models.rs b/src/models/engine_models.rs index 3108e6e..4d56836 100644 --- a/src/models/engine_models.rs +++ b/src/models/engine_models.rs @@ -86,6 +86,42 @@ pub trait SearchEngine: Sync + Send { .change_context(EngineError::RequestError)?) } + /// This helper function fetches/requests the json search results from the upstream search engine as a vector of bytes. + /// + /// # Arguments + /// + /// * `url` - It takes the url of the upstream search engine with the user requested search + /// query appended in the search parameters. + /// * `header_map` - It takes the http request headers to be sent to the upstream engine in + /// order to prevent being detected as a bot. It takes the header as a HeaderMap type. + /// * `request_timeout` - It takes the request timeout value as seconds which is used to limit + /// the amount of time for each request to remain connected when until the results can be provided + /// by the upstream engine. + /// + /// # Error + /// + /// It returns the html data as a vector of bytes if the upstream engine provides the data as expected + /// otherwise it returns a custom `EngineError`. + async fn fetch_json_as_bytes_from_upstream( + &self, + url: &str, + header_map: reqwest::header::HeaderMap, + client: &Client, + ) -> Result, EngineError> { + // fetch the json response from upstream search engine + + Ok(client + .get(url) + .headers(header_map) // add spoofed headers to emulate human behavior + .send() + .await + .change_context(EngineError::RequestError)? + .bytes() + .await + .change_context(EngineError::RequestError)? + .to_vec()) + } + /// This function scrapes results from the upstream engine and puts all the scraped results like /// title, visiting_url (href in html),engine (from which engine it was fetched from) and description /// in a RawSearchResult and then adds that to HashMap whose keys are url and values are RawSearchResult From a92550e050dc6bc08fb53ef4c96f785b327cb752 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 31 Jan 2024 02:08:37 -0500 Subject: [PATCH 27/37] =?UTF-8?q?=F0=9F=91=B7=20GitHub=20action=20to=20aut?= =?UTF-8?q?omate=20release=20based=20on=20semantic=20versioning=20(#499)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: add release.yml to help automate release versions * ci: add release.yml to help automate release versions - edited file further to cover edge cases * ci: add release.yml to help automate release versions - I had a typo in a line which pointed at a different variable * ci: add release.yml to help automate release versions - Fixing the release portion to prepend a v * ci: add release.yml to help automate release versions. Lowercased V so it looks like v1.1.0 instead of V1.1.0 * ci: changing structure of release.yml to be easier to understand * ci: #minor changes made * ci: #minor changes made to make rolling and stable both release branches * ci: #minor changes to hopefully turn off pre-release * ci: #minor change - trying to make pre-release suffic set to nothing * ci: #minor change - removed older release action for a better up to date one * ci: #minor change * ci: #minor change to fixed the pre-release issue * ci: #minor change to fixed the pre-release issue again * ci: #minor change to fixed the pre-release issue again2 * ci: changed action for tag. I believe it was causing errors * ci: missing } * ci: testing some things * ci: fixing self inflicted error where I forgot to put an ID in * ci: fixing to make the release branch rolling * major: testing updating major * BREAKING CHANGE: removed changelog action * ci: reset commit and removed changelog * ci: changed action for tag. I believe it was causing errors * ci: missing } * ci: testing some things * ci: fixing self inflicted error where I forgot to put an ID in * ci: fixing to make the release branch rolling * ci: reset commit and removed changelog * ci: added step to change Cargo.toml app version to the latest tag version without the v * ci: using echo and double quotes in the appversion variable. Testing now * ci: testing autocommit * Apply automatic changes * ci: testing autocommit further * ci: testing autocommit: removing tagging since it creates another tag instead of using that tag * Apply automatic changes * ci: testing autocommit with branch protection and doing a PR to rolling * Apply automatic changes * ci: testing PR create and merge - removed the if that was causing it not to run * ci: testing PR create and merge - removed the if that was causing it not to run * Apply automatic changes * ci: testing PR create and merge - fixed permission issues * ci: testing PR create and merge - fixed permission issues * Apply automatic changes * ci: testing PR create and merge - potentially fixed PR issue * Apply automatic changes * ci: testing PR create and merge - potentially fixed PR issue2 * ci: testing PR create and merge - potentially fixed PR issue + fixing what branch it targets * ci: testing PR create and merge - testing PAT * ci: testing PR create and merge - testing to see if merge loop stops * ci: adding in cargo.toml version in auto git commit * Update Cargo.toml Co-authored-by: alamin655 <129589283+alamin655@users.noreply.github.com> * ci: adding in steps to regenerate the Cargo.lock file * ci: adding in steps to regenerate the Cargo.lock file - fixing commit issue with cargo.toml --------- Co-authored-by: scottc943 Co-authored-by: alamin655 <129589283+alamin655@users.noreply.github.com> --- .github/workflows/release.yml | 72 +++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..57c5100 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,72 @@ +name: Bump release version +on: + pull_request: + branches: [rolling] + types: + - closed + +permissions: + contents: write + pull-requests: write + repository-projects: write + +concurrency: production + +jobs: + build: + name: bump tag version and release + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + - name: Bump version and push tag + id: version-bump + uses: hennejg/github-tag-action@v4.3.1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + release_branches: rolling + - name: create branch + uses: peterjgrainger/action-create-branch@v2.2.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + branch: update-from-${{ github.sha }} + - name: update cargo.toml + run: | + appversion=$(echo "${{ steps.version-bump.outputs.new_tag }}" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+') + sed -i -e "s/^version = .*/version = \"$appversion\"/" Cargo.toml + - run: rustup toolchain install stable --profile minimal + - run: rustup update stable && rustup default stable + - name: regenerate cargo.lock + run: cargo generate-lockfile + - name: auto commit + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "[skip ci] updating app version to ${{ steps.version-bump.outputs.new_tag }}" + branch: update-from-${{ github.sha }} + # create PR using GitHub CLI + - name: create PR with update info + id: create-pr + run: gh pr create --base rolling --head update-from-${{ github.sha }} --title 'Merge new update into rolling' --body 'Created by Github action' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # merge PR using GitHub CLI + - name: merge PR with update info + id: merge-pr + run: gh pr merge --admin --merge --subject 'Merge update info' --delete-branch + env: + GH_TOKEN: ${{ secrets.ADMIN_RIGHTS_TOKEN }} + - name: Create Release + uses: softprops/action-gh-release@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + generate_release_notes: true + name: ${{ steps.version-bump.outputs.new_tag }} + tag_name: ${{ steps.version-bump.outputs.new_tag }} + prerelease: false + env: + GITHUB_REPOSITORY: ${{ github.repository }} \ No newline at end of file From ca425f9ef5ac55198ffacdf151fd967e85afbef3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 06:39:32 +0000 Subject: [PATCH 28/37] build(deps): bump hennejg/github-tag-action from 4.3.1 to 4.4.0 (#507) --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 57c5100..4ec7477 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,7 +25,7 @@ jobs: fetch-depth: 0 - name: Bump version and push tag id: version-bump - uses: hennejg/github-tag-action@v4.3.1 + uses: hennejg/github-tag-action@v4.4.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} release_branches: rolling From 1909cc36a848f8bf2b0adbc53c314c63fbc3bfb1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 06:44:35 +0000 Subject: [PATCH 29/37] build(deps): bump actions/cache from 3 to 4 (#508) --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 37d0eb3..1220fd9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -34,7 +34,7 @@ jobs: uses: docker/setup-buildx-action@v3 # Set buildx cache - name: Cache register - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /tmp/.buildx-cache key: buildx-cache From 57267827f5a7169010cc66d4c78e43ec030535a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 06:53:56 +0000 Subject: [PATCH 30/37] build(deps): bump peterjgrainger/action-create-branch (#509) --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4ec7477..a907710 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,7 +30,7 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} release_branches: rolling - name: create branch - uses: peterjgrainger/action-create-branch@v2.2.0 + uses: peterjgrainger/action-create-branch@v2.4.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: From 6c3d9ecd503cd536143da6747d4be4469d15246b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 10:10:08 +0300 Subject: [PATCH 31/37] :arrow_up: build(deps): bump serde from 1.0.193 to 1.0.196 (#510) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.193 to 1.0.196. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.193...v1.0.196) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: neon_arch --- Cargo.lock | 146 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 74 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 72af913..e1abead 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,8 +111,8 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ - "quote 1.0.33", - "syn 2.0.43", + "quote 1.0.35", + "syn 2.0.48", ] [[package]] @@ -223,9 +223,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1f50ebbb30eca122b188319a4398b3f7bb4a8cdf50ecfb73bfc6a3c3ce54f5" dependencies = [ "actix-router", - "proc-macro2 1.0.71", - "quote 1.0.33", - "syn 2.0.43", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", ] [[package]] @@ -378,9 +378,9 @@ version = "0.1.76" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", - "syn 2.0.43", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", ] [[package]] @@ -744,8 +744,8 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e1e0fdd2e5d3041e530e1b21158aeeef8b5d0e306bc5c1e3d6cf0930d10e25a" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", "syn 1.0.109", ] @@ -987,8 +987,8 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ - "quote 1.0.33", - "syn 2.0.43", + "quote 1.0.35", + "syn 2.0.48", ] [[package]] @@ -1026,8 +1026,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -1170,8 +1170,8 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", "syn 1.0.109", "synstructure", ] @@ -1341,9 +1341,9 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", - "syn 2.0.43", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", ] [[package]] @@ -1554,8 +1554,8 @@ dependencies = [ "log", "mac", "markup5ever 0.11.0", - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", "syn 1.0.109", ] @@ -2016,8 +2016,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0be95d66c3024ffce639216058e5bae17a83ecaf266ffc6e4d060ad447c9eed2" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", "syn 1.0.109", ] @@ -2301,9 +2301,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", - "syn 2.0.43", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", ] [[package]] @@ -2512,9 +2512,9 @@ checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ "phf_generator 0.11.2", "phf_shared 0.11.2", - "proc-macro2 1.0.71", - "quote 1.0.33", - "syn 2.0.43", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", ] [[package]] @@ -2559,9 +2559,9 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", - "syn 2.0.43", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", ] [[package]] @@ -2618,8 +2618,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", "syn 1.0.109", "version_check", ] @@ -2630,8 +2630,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", "version_check", ] @@ -2646,9 +2646,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -2701,11 +2701,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ - "proc-macro2 1.0.71", + "proc-macro2 1.0.78", ] [[package]] @@ -3237,22 +3237,22 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", - "syn 2.0.43", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", ] [[package]] @@ -3441,8 +3441,8 @@ checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" dependencies = [ "phf_generator 0.7.24", "phf_shared 0.7.24", - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", "string_cache_shared", ] @@ -3454,8 +3454,8 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", ] [[package]] @@ -3487,19 +3487,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.43" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", "unicode-ident", ] @@ -3509,8 +3509,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", + "proc-macro2 1.0.78", + "quote 1.0.35", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -3733,9 +3733,9 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", - "syn 2.0.43", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", ] [[package]] @@ -4081,9 +4081,9 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.71", - "quote 1.0.33", - "syn 2.0.43", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -4105,7 +4105,7 @@ version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" dependencies = [ - "quote 1.0.33", + "quote 1.0.35", "wasm-bindgen-macro-support", ] @@ -4115,9 +4115,9 @@ version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", - "syn 2.0.43", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4417,9 +4417,9 @@ version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ - "proc-macro2 1.0.71", - "quote 1.0.33", - "syn 2.0.43", + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fdfbe32..9006af9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ path = "src/bin/websurfx.rs" [dependencies] reqwest = {version="0.11.22", default-features=false, features=["rustls-tls","brotli", "gzip"]} tokio = {version="1.32.0",features=["rt-multi-thread","macros"], default-features = false} -serde = {version="1.0.190", default-features=false, features=["derive"]} +serde = {version="1.0.196", default-features=false, features=["derive"]} serde_json = {version="1.0.109", default-features=false} maud = {version="0.25.0", default-features=false, features=["actix-web"]} scraper = {version="0.18.1", default-features = false} From 3d76b1bd8678fa9940c96d9d27059890776ceb43 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 07:14:54 +0000 Subject: [PATCH 32/37] build(deps): bump lightningcss from 1.0.0-alpha.51 to 1.0.0-alpha.52 (#513) --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1abead..c711d20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1860,9 +1860,9 @@ dependencies = [ [[package]] name = "lightningcss" -version = "1.0.0-alpha.51" +version = "1.0.0-alpha.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d6ad516c08b24c246b339159dc2ee2144c012e8ebdf4db4bddefb8734b2b69" +checksum = "771a62dedf5ec563bbfea9760f6c6a6bc546e67355eba0cd7d00c0dc34b11d90" dependencies = [ "ahash 0.7.7", "bitflags 2.4.1", diff --git a/Cargo.toml b/Cargo.toml index 9006af9..f8bfec1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,7 @@ criterion = {version="0.5.1", default-features=false} tempfile = {version="3.8.0", default-features=false} [build-dependencies] -lightningcss = {version="1.0.0-alpha.50", default-features=false, features=["grid"]} +lightningcss = {version="1.0.0-alpha.52", default-features=false, features=["grid"]} minify-js = {version="0.6.0", default-features=false} [profile.dev] From ae5b3370bc9bf4dcde69d8822d50f3414c329149 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 07:20:11 +0000 Subject: [PATCH 33/37] build(deps): bump smallvec from 1.11.2 to 1.13.1 (#514) --- Cargo.lock | 28 ++++++++++++++-------------- Cargo.toml | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c711d20..31142b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,7 +31,7 @@ dependencies = [ "futures-util", "log", "once_cell", - "smallvec 1.11.2", + "smallvec 1.13.1", ] [[package]] @@ -99,7 +99,7 @@ dependencies = [ "pin-project-lite", "rand 0.8.5", "sha1", - "smallvec 1.11.2", + "smallvec 1.13.1", "tokio 1.35.1", "tokio-util", "tracing", @@ -210,7 +210,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.1", - "smallvec 1.11.2", + "smallvec 1.13.1", "socket2", "time 0.3.31", "url 2.5.0", @@ -956,7 +956,7 @@ dependencies = [ "dtoa-short", "itoa 1.0.10", "phf 0.11.2", - "smallvec 1.11.2", + "smallvec 1.13.1", ] [[package]] @@ -969,7 +969,7 @@ dependencies = [ "dtoa-short", "itoa 1.0.10", "phf 0.11.2", - "smallvec 1.11.2", + "smallvec 1.13.1", ] [[package]] @@ -1448,7 +1448,7 @@ dependencies = [ "parking_lot 0.12.1", "quanta", "rand 0.8.5", - "smallvec 1.11.2", + "smallvec 1.13.1", ] [[package]] @@ -1875,7 +1875,7 @@ dependencies = [ "parcel_selectors", "paste", "pathdiff", - "smallvec 1.11.2", + "smallvec 1.13.1", ] [[package]] @@ -2077,7 +2077,7 @@ dependencies = [ "crossbeam-utils 0.8.18", "dashmap", "skeptic", - "smallvec 1.11.2", + "smallvec 1.13.1", "tagptr", "triomphe", ] @@ -2337,7 +2337,7 @@ dependencies = [ "phf 0.10.1", "phf_codegen 0.10.0", "precomputed-hash", - "smallvec 1.11.2", + "smallvec 1.13.1", ] [[package]] @@ -2385,7 +2385,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall 0.4.1", - "smallvec 1.11.2", + "smallvec 1.13.1", "windows-targets 0.48.5", ] @@ -3208,7 +3208,7 @@ dependencies = [ "phf_codegen 0.10.0", "precomputed-hash", "servo_arc", - "smallvec 1.11.2", + "smallvec 1.13.1", ] [[package]] @@ -3366,9 +3366,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" dependencies = [ "serde", ] @@ -4180,7 +4180,7 @@ dependencies = [ "scraper", "serde", "serde_json", - "smallvec 1.11.2", + "smallvec 1.13.1", "tempfile", "tokio 1.35.1", ] diff --git a/Cargo.toml b/Cargo.toml index f8bfec1..a178871 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ blake3 = {version="1.5.0", default-features=false} error-stack = {version="0.4.0", default-features=false, features=["std"]} async-trait = {version="0.1.76", default-features=false} regex = {version="1.9.4", features=["perf"], default-features = false} -smallvec = {version="1.11.0", features=["union", "serde"], default-features=false} +smallvec = {version="1.13.1", features=["union", "serde"], default-features=false} futures = {version="0.3.28", default-features=false} dhat = {version="0.3.2", optional = true, default-features=false} mimalloc = { version = "0.1.38", default-features = false } From 0f19ade40c4a6363b63b4c39f3aefd355abbe575 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 07:24:55 +0000 Subject: [PATCH 34/37] build(deps): bump env_logger from 0.10.1 to 0.11.1 (#512) --- Cargo.lock | 16 +++++++++++++--- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31142b5..a280d39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1101,14 +1101,24 @@ dependencies = [ ] [[package]] -name = "env_logger" -version = "0.10.1" +name = "env_filter" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" dependencies = [ "log", ] +[[package]] +name = "env_logger" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e7cf40684ae96ade6232ed84582f40ce0a66efcd43a5117aef610534f8e0b8" +dependencies = [ + "env_filter", + "log", +] + [[package]] name = "envmnt" version = "0.8.4" diff --git a/Cargo.toml b/Cargo.toml index a178871..fd8e86a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ actix-web = {version="4.4.0", features = ["cookies", "macros", "compress-brotli" actix-files = {version="0.6.2", default-features=false} actix-cors = {version="0.6.4", default-features=false} fake-useragent = {version="0.1.3", default-features=false} -env_logger = {version="0.10.0", default-features=false} +env_logger = {version="0.11.1", default-features=false} log = {version="0.4.20", default-features=false} mlua = {version="0.9.1", features=["luajit", "vendored"], default-features=false} redis = {version="0.24.0", features=["tokio-comp","connection-manager"], default-features = false, optional = true} From 8790f5f719e21292e5d74cbd405c5eb0f45add58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 07:29:35 +0000 Subject: [PATCH 35/37] build(deps): bump actix-files from 0.6.2 to 0.6.5 (#511) --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a280d39..2d9e6f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,16 +36,15 @@ dependencies = [ [[package]] name = "actix-files" -version = "0.6.2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d832782fac6ca7369a70c9ee9a20554623c5e51c76e190ad151780ebea1cf689" +checksum = "bf0bdd6ff79de7c9a021f5d9ea79ce23e108d8bfc9b49b5b4a2cf6fad5a35212" dependencies = [ "actix-http", "actix-service", "actix-utils", "actix-web", - "askama_escape", - "bitflags 1.3.2", + "bitflags 2.4.1", "bytes 1.5.0", "derive_more", "futures-core", @@ -55,6 +54,7 @@ dependencies = [ "mime_guess", "percent-encoding 2.3.1", "pin-project-lite", + "v_htmlescape", ] [[package]] @@ -346,12 +346,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" -[[package]] -name = "askama_escape" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" - [[package]] name = "async-compression" version = "0.4.5" @@ -4018,6 +4012,12 @@ dependencies = [ "rand 0.6.5", ] +[[package]] +name = "v_htmlescape" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e8257fbc510f0a46eb602c10215901938b5c2a7d5e70fc11483b1d3c9b5b18c" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index fd8e86a..b3d3cb3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ serde_json = {version="1.0.109", default-features=false} maud = {version="0.25.0", default-features=false, features=["actix-web"]} scraper = {version="0.18.1", default-features = false} actix-web = {version="4.4.0", features = ["cookies", "macros", "compress-brotli"], default-features=false} -actix-files = {version="0.6.2", default-features=false} +actix-files = {version="0.6.5", default-features=false} actix-cors = {version="0.6.4", default-features=false} fake-useragent = {version="0.1.3", default-features=false} env_logger = {version="0.11.1", default-features=false} From 280c7e2b5e249a9f6417531fd449a407a034976e Mon Sep 17 00:00:00 2001 From: Scott Date: Fri, 2 Feb 2024 22:17:38 -0500 Subject: [PATCH 36/37] :bug: Auto release `github` action failure due to missing permissions (#506) * ci: add release.yml to help automate release versions * ci: add release.yml to help automate release versions - edited file further to cover edge cases * ci: add release.yml to help automate release versions - I had a typo in a line which pointed at a different variable * ci: add release.yml to help automate release versions - Fixing the release portion to prepend a v * ci: add release.yml to help automate release versions. Lowercased V so it looks like v1.1.0 instead of V1.1.0 * ci: changing structure of release.yml to be easier to understand * ci: #minor changes made * ci: #minor changes made to make rolling and stable both release branches * ci: #minor changes to hopefully turn off pre-release * ci: #minor change - trying to make pre-release suffic set to nothing * ci: #minor change - removed older release action for a better up to date one * ci: #minor change * ci: #minor change to fixed the pre-release issue * ci: #minor change to fixed the pre-release issue again * ci: #minor change to fixed the pre-release issue again2 * ci: changed action for tag. I believe it was causing errors * ci: missing } * ci: testing some things * ci: fixing self inflicted error where I forgot to put an ID in * ci: fixing to make the release branch rolling * major: testing updating major * BREAKING CHANGE: removed changelog action * ci: reset commit and removed changelog * ci: changed action for tag. I believe it was causing errors * ci: missing } * ci: testing some things * ci: fixing self inflicted error where I forgot to put an ID in * ci: fixing to make the release branch rolling * ci: reset commit and removed changelog * ci: added step to change Cargo.toml app version to the latest tag version without the v * ci: using echo and double quotes in the appversion variable. Testing now * ci: testing autocommit * Apply automatic changes * ci: testing autocommit further * ci: testing autocommit: removing tagging since it creates another tag instead of using that tag * Apply automatic changes * ci: testing autocommit with branch protection and doing a PR to rolling * Apply automatic changes * ci: testing PR create and merge - removed the if that was causing it not to run * ci: testing PR create and merge - removed the if that was causing it not to run * Apply automatic changes * ci: testing PR create and merge - fixed permission issues * ci: testing PR create and merge - fixed permission issues * Apply automatic changes * ci: testing PR create and merge - potentially fixed PR issue * Apply automatic changes * ci: testing PR create and merge - potentially fixed PR issue2 * ci: testing PR create and merge - potentially fixed PR issue + fixing what branch it targets * ci: testing PR create and merge - testing PAT * ci: testing PR create and merge - testing to see if merge loop stops * ci: adding in cargo.toml version in auto git commit * Update Cargo.toml Co-authored-by: alamin655 <129589283+alamin655@users.noreply.github.com> * ci: adding in steps to regenerate the Cargo.lock file * ci: adding in steps to regenerate the Cargo.lock file - fixing commit issue with cargo.toml * ci: testing permissions * ci: testing permissions with PAT * ci: testing permissions with PAT again --------- Co-authored-by: scottc943 Co-authored-by: alamin655 <129589283+alamin655@users.noreply.github.com> --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a907710..3b9d8db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,12 +27,12 @@ jobs: id: version-bump uses: hennejg/github-tag-action@v4.4.0 with: - github_token: ${{ secrets.GITHUB_TOKEN }} + github_token: ${{ secrets.ADMIN_RIGHTS_TOKEN }} release_branches: rolling - name: create branch uses: peterjgrainger/action-create-branch@v2.4.0 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.ADMIN_RIGHTS_TOKEN }} with: branch: update-from-${{ github.sha }} - name: update cargo.toml @@ -53,7 +53,7 @@ jobs: id: create-pr run: gh pr create --base rolling --head update-from-${{ github.sha }} --title 'Merge new update into rolling' --body 'Created by Github action' env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.ADMIN_RIGHTS_TOKEN }} # merge PR using GitHub CLI - name: merge PR with update info id: merge-pr @@ -63,7 +63,7 @@ jobs: - name: Create Release uses: softprops/action-gh-release@v1 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.ADMIN_RIGHTS_TOKEN }} generate_release_notes: true name: ${{ steps.version-bump.outputs.new_tag }} tag_name: ${{ steps.version-bump.outputs.new_tag }} From 2df6499fb2d3b2064599f4177d4be581316e15ca Mon Sep 17 00:00:00 2001 From: alamin655 <129589283+alamin655@users.noreply.github.com> Date: Thu, 8 Feb 2024 10:22:16 +0530 Subject: [PATCH 37/37] =?UTF-8?q?=F0=9F=93=9D=20docs(instances):=20more=20?= =?UTF-8?q?instances=20with=20the=20rolling/edge=20version=20(#495)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add 2 additional instance Added two new instance - https://websurfx.instance.pp.ua - https://alamin655-spacex.hf.space Changed the `Managed by` column to `Status` * πŸ”– chore(release): bump the app version (#495) * πŸ”– chore(release): bump the app version (#495) * πŸ”– chore(release): bump the app version (#495) * πŸ”– chore(release): bump the app version (#495) * Remove unnecessary line Co-authored-by: neon_arch * Update docs/instances.md Co-authored-by: neon_arch * Update instances.md * bump version * bump version * Update instances.md * Update instances.md * use shields.io * Update instances.md * change websurfx.co to websurfx.pp.ua * update the status badge * bump app version v1.9.20 * Add maintained by Co-authored-by: neon_arch --------- Co-authored-by: neon_arch --- Cargo.lock | 2 +- Cargo.toml | 2 +- docs/instances.md | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d9e6f9..e2b8de1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4156,7 +4156,7 @@ checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" [[package]] name = "websurfx" -version = "1.9.4" +version = "1.9.20" dependencies = [ "actix-cors", "actix-files", diff --git a/Cargo.toml b/Cargo.toml index b3d3cb3..fbb173c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "1.9.4" +version = "1.9.20" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" diff --git a/docs/instances.md b/docs/instances.md index 970e372..0e86128 100644 --- a/docs/instances.md +++ b/docs/instances.md @@ -4,10 +4,12 @@ This page provides a list of `Websurfx` instances provided by us and our community. -|URL|Network|Version|Location|Behind Cloudflare?|Maintained By|TLS|IPv6|Comment| +|URL|Network|Version|Location|Status|Maintained By|TLS|IPv6|Comment| |-|-|-|-|-|-|-|-|-| -|https://websurfx.co/|www|edge|πŸ‡ΊπŸ‡Έ US|||βœ…|❌|| -|https://websurfx.onrender.com/|www|edge|πŸ‡ΊπŸ‡Έ US|||βœ…|❌|| -|https://alamin655-websurfx.hf.space/|www|v0.21.4|πŸ‡ΊπŸ‡Έ US||[websurfx project](https://github.com/neon-mmd/websurfx)|βœ…|❌|| +|https://websurfx.pp.ua|www|rolling|πŸ‡ΊπŸ‡Έ US||[Websurfx Project](https://github.com/neon-mmd/websurfx)|βœ…|βœ…|| +|https://alamin655-spacex.hf.space|www|rolling|πŸ‡ΊπŸ‡Έ US||[Websurfx Project](https://github.com/neon-mmd/websurfx)|βœ…|❌|| +|https://websurfx.instance.pp.ua|www|rolling|πŸ‡ΊπŸ‡Έ US||[Websurfx Project](https://github.com/neon-mmd/websurfx)|βœ…|βœ…|| +|https://alamin655-surfx.hf.space|www|stable|πŸ‡ΊπŸ‡Έ US||[Websurfx Project](https://github.com/neon-mmd/websurfx)|βœ…|❌|| + [⬅️ Go back to Home](./README.md)