From 3c7edb80d0663365c5828b95bf909af87bad857d Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Mon, 15 May 2023 00:20:43 +0000 Subject: [PATCH 001/392] refactor: reduce connections created with RedisCache --- .gitignore | 2 ++ src/cache/cacher.rs | 36 ++++++++++++++++-------------------- src/server/routes.rs | 12 +++++------- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index ea8c4bf..c39800b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /target + +dump.rdb \ No newline at end of file diff --git a/src/cache/cacher.rs b/src/cache/cacher.rs index 54d9a48..87a6c6d 100644 --- a/src/cache/cacher.rs +++ b/src/cache/cacher.rs @@ -10,9 +10,8 @@ use redis::{Client, Commands, Connection}; /// # Fields /// /// * `redis_connection_url` - It stores the redis Connection url address. -#[derive(Clone)] pub struct RedisCache { - redis_connection_url: String, + connection: Connection, } impl RedisCache { @@ -21,10 +20,11 @@ impl RedisCache { /// # Arguments /// /// * `redis_connection_url` - It stores the redis Connection url address. - pub fn new(redis_connection_url: String) -> Self { - RedisCache { - redis_connection_url, - } + pub fn new(redis_connection_url: String) -> Result> { + let client = Client::open(redis_connection_url)?; + let connection = client.get_connection()?; + let redis_cache = RedisCache { connection }; + Ok(redis_cache) } /// A helper function which computes the hash of the url and formats and returns it as string. @@ -32,7 +32,7 @@ impl RedisCache { /// # Arguments /// /// * `url` - It takes an url as string. - fn compute_url_hash(self, url: &str) -> String { + fn compute_url_hash(url: &str) -> String { format!("{:?}", compute(url)) } @@ -41,11 +41,9 @@ impl RedisCache { /// # Arguments /// /// * `url` - It takes an url as a string. - pub fn cached_results_json(self, url: String) -> Result> { - let hashed_url_string = self.clone().compute_url_hash(&url); - let mut redis_connection: Connection = - Client::open(self.redis_connection_url)?.get_connection()?; - Ok(redis_connection.get(hashed_url_string)?) + pub fn cached_results_json(&mut self, url: &str) -> Result> { + let hashed_url_string = Self::compute_url_hash(url); + Ok(self.connection.get(hashed_url_string)?) } /// A function which caches the results by using the hashed `url` as the key and @@ -57,20 +55,18 @@ impl RedisCache { /// * `json_results` - It takes the json results string as an argument. /// * `url` - It takes the url as a String. pub fn cache_results( - self, + &mut self, json_results: String, - url: String, + url: &str, ) -> Result<(), Box> { - let hashed_url_string = self.clone().compute_url_hash(&url); - let mut redis_connection: Connection = - Client::open(self.redis_connection_url)?.get_connection()?; + let hashed_url_string = Self::compute_url_hash(url); // put results_json into cache - redis_connection.set(hashed_url_string.clone(), json_results)?; + self.connection.set(&hashed_url_string, json_results)?; // Set the TTL for the key to 60 seconds - redis_connection - .expire::(hashed_url_string.clone(), 60) + self.connection + .expire::(hashed_url_string, 60) .unwrap(); Ok(()) diff --git a/src/server/routes.rs b/src/server/routes.rs index 1ee9f35..e97bc2b 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -73,7 +73,7 @@ pub async fn search( let params = web::Query::::from_query(req.query_string())?; //Initialize redis cache connection struct - let redis_cache = RedisCache::new(config.redis_connection_url.clone()); + let mut redis_cache = RedisCache::new(config.redis_connection_url.clone())?; match ¶ms.q { Some(query) => { if query.trim().is_empty() { @@ -117,7 +117,7 @@ pub async fn search( }; // fetch the cached results json. - let cached_results_json = redis_cache.clone().cached_results_json(page_url.clone()); + let cached_results_json = redis_cache.cached_results_json(&page_url); // check if fetched results was indeed fetched or it was an error and if so // handle the data accordingly. match cached_results_json { @@ -128,12 +128,10 @@ pub async fn search( } Err(_) => { let mut results_json: crate::search_results_handler::aggregation_models::SearchResults = - aggregate(query, page).await?; + aggregate(query, page).await?; results_json.add_style(config.style.clone()); - redis_cache.clone().cache_results( - serde_json::to_string(&results_json)?, - page_url.clone(), - )?; + redis_cache + .cache_results(serde_json::to_string(&results_json)?, &page_url)?; let page_content: String = hbs.render("search", &results_json)?; Ok(HttpResponse::Ok().body(page_content)) } From 019b3322e79b5e549bf12222aa4964c43dae3154 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 16 May 2023 12:22:00 +0300 Subject: [PATCH 002/392] chore: add enum to handle different reqwest errors & add timeout to requests --- src/engines/duckduckgo.rs | 3 ++- src/engines/engine_models.rs | 8 ++++++++ src/engines/mod.rs | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 src/engines/engine_models.rs diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index 254ab16..7232165 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -2,7 +2,7 @@ //! by querying the upstream duckduckgo search engine with user provided query and with a page //! number if provided. -use std::collections::HashMap; +use std::{collections::HashMap, time::Duration}; use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT}; use scraper::{Html, Selector}; @@ -57,6 +57,7 @@ pub async fn results( // TODO: Write better error handling code to handle no results case. let results: String = reqwest::Client::new() .get(url) + .timeout(Duration::from_secs(30)) .headers(header_map) // add spoofed headers to emulate human behaviour .send() .await? diff --git a/src/engines/engine_models.rs b/src/engines/engine_models.rs new file mode 100644 index 0000000..7dc8ad6 --- /dev/null +++ b/src/engines/engine_models.rs @@ -0,0 +1,8 @@ +#[derive(Debug)] +pub enum ReqwestError{ + NotFound, + Timeout, + Forbidden, + AccessDenied, + TooManyRequests +} diff --git a/src/engines/mod.rs b/src/engines/mod.rs index 7f390b1..f9bb8ad 100644 --- a/src/engines/mod.rs +++ b/src/engines/mod.rs @@ -1,2 +1,3 @@ pub mod duckduckgo; +pub mod engine_models; pub mod searx; From 2fc3ab42fe849af780a61d10cafe4b74bdda9663 Mon Sep 17 00:00:00 2001 From: B C SAMRUDH <114090255+bcsamrudh@users.noreply.github.com> Date: Wed, 17 May 2023 22:53:10 +0530 Subject: [PATCH 003/392] Added an about page. --- public/templates/about.html | 39 +++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/public/templates/about.html b/public/templates/about.html index 56c2165..9c4cbb0 100644 --- a/public/templates/about.html +++ b/public/templates/about.html @@ -1,20 +1,29 @@ {{>header this}}
-

Websurfx

- a lightening fast, privacy respecting, secure meta search engine -
- Lorem ipsum dolor sit amet, officia excepteur ex fugiat reprehenderit enim - labore culpa sint ad nisi Lorem pariatur mollit ex esse exercitation amet. - Nisi anim cupidatat excepteur officia. Reprehenderit nostrud nostrud ipsum - Lorem est aliquip amet voluptate voluptate dolor minim nulla est proident. - Nostrud officia pariatur ut officia. Sit irure elit esse ea nulla sunt ex - occaecat reprehenderit commodo officia dolor Lorem duis laboris cupidatat - officia voluptate. Culpa proident adipisicing id nulla nisi laboris ex in - Lorem sunt duis officia eiusmod. Aliqua reprehenderit commodo ex non - excepteur duis sunt velit enim. Voluptate laboris sint cupidatat ullamco ut - ea consectetur et est culpa et culpa duis. +
+
+

Websurfx

+
+
+

A modern-looking, lightning-fast, privacy-respecting, secure meta search engine written in Rust. It provides a fast and secure search experience while respecting user privacy.
It aggregates results from multiple search engines and presents them in an unbiased manner, filtering out trackers and ads. +

+ +

Some of the Top Features:

+ +
    Lightning fast - Results load within milliseconds for an instant search experience.
+ +
    Secure search - All searches are performed over an encrypted connection to prevent snooping.
+ +
    Ad free results - All search results are ad free and clutter free for a clean search experience.
+ +
    Privacy focused - Websurface does not track, store or sell your search data. Your privacy is our priority.
+ +
    Free and Open source - The entire project's code is open source and available for free on GitHub under an GNU Affero General Public License.
+ +
    Highly customizable - Websurface comes with 9 built-in color themes and supports creating custom themes effortlessly.
+ +

Devoloped by: Websurfx team

{{>footer}} + From 0a947cd400878658a40cdd4bdbfe91fb36305e56 Mon Sep 17 00:00:00 2001 From: B C SAMRUDH <114090255+bcsamrudh@users.noreply.github.com> Date: Wed, 17 May 2023 22:54:16 +0530 Subject: [PATCH 004/392] Added the CSS code for about.html --- public/static/themes/simple.css | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/public/static/themes/simple.css b/public/static/themes/simple.css index 97643e8..b9390e7 100644 --- a/public/static/themes/simple.css +++ b/public/static/themes/simple.css @@ -260,3 +260,40 @@ footer { .page_navigation button:active { filter: brightness(1.2); } + +/* Styles for the about page */ + + .about-container article{ + font-size: 1.5rem; + color:var(--fg); + padding-bottom: 10px; + } + + .about-container article h1{ + color: var(--2); + font-size: 2.8rem; + } + + .about-container article div{ + padding-bottom: 15px; + } + + .about-container a{ + color:var(--3); + } + + .about-container article h2{ + color: var(--3); + font-size: 1.8rem; + padding-bottom: 10px; + } + + .about-container p{ + color:var(--fg); + font-size: 1.6rem; + padding-bottom: 10px; + } + + .about-container h3{ + font-size: 1.5rem; + } From 85e5868ac38fcf2516705c033677e695f22b79e3 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Thu, 18 May 2023 10:12:51 +0000 Subject: [PATCH 005/392] ci: remove releases automation github action --- .github/workflows/releases.yml | 74 ---------------------------------- 1 file changed, 74 deletions(-) delete mode 100644 .github/workflows/releases.yml diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml deleted file mode 100644 index e46c5aa..0000000 --- a/.github/workflows/releases.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Releases -on: - push: - branches: - - "rolling" - -concurrency: - group: "rolling-branch" - -jobs: - changelog: - if: github.repository == 'neon-mmd/websurfx' - runs-on: ubuntu-latest - - steps: - # Create a temporary, uniquely named branch to push release info to - - name: create temporary branch - uses: peterjgrainger/action-create-branch@v2.3.0 - id: create-branch - with: - branch: "release-from-${{ github.sha }}" - sha: "${{ github.sha }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # check out the repository afterwards - - uses: actions/checkout@v3 - - # fetch branches and switch to the temporary branch - - name: switch to new branch - run: git fetch --all && git checkout --track origin/release-from-${{ github.sha }} - - # update app config with version - - name: get-npm-version - id: package-version - uses: martinbeentjes/npm-get-version-action@master - - name: update app config - run: sed -i 's/0.0.0/${{ steps.package-version.outputs.current-version}}/g' config/app.json - - # create release info and push it upstream - - name: conventional Changelog Action - id: changelog - uses: TriPSs/conventional-changelog-action@v3 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - version-file: "./Cargo.toml" - git-branch: "release-from-${{ github.sha }}" - skip-on-empty: false - skip-git-pull: true - - # create PR using GitHub CLI - - name: create PR with release info - id: create-pr - run: gh pr create --base main --head release-from-${{ github.sha }} --title 'Merge new release into rolling' --body 'Created by Github action' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # merge PR using GitHub CLI - - name: merge PR with release info - id: merge-pr - run: gh pr merge --admin --merge --subject 'Merge release info' --delete-branch - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # release info is now in main so we can continue as before - - name: create release with last commit - uses: actions/create-release@v1 - if: steps.changelog.outputs.skipped == 'false' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ steps.changelog.outputs.tag }} - release_name: ${{ steps.changelog.outputs.tag }} - body: ${{ steps.changelog.outputs.clean_changelog }} From 9a204b2f985c29020d8a177ed2894b98a1b4adc5 Mon Sep 17 00:00:00 2001 From: MD AL AMIN TALUKDAR <129589283+alamin655@users.noreply.github.com> Date: Fri, 19 May 2023 17:13:11 +0530 Subject: [PATCH 006/392] Fix page_url assignment in search route The page_url variable in the search route was not being properly assigned in certain cases. This commit fixes the issue by ensuring that page_url is assigned the correct value based on the search parameters. In the match expression, the conditions have been adjusted to correctly handle the page number and construct the appropriate page_url. This ensures that the generated URL for the search page is accurate and reflects the search query and page number. This change improves the functionality and reliability of the search route by correctly setting the page_url variable based on the provided search parameters. --- src/server/routes.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/server/routes.rs b/src/server/routes.rs index e97bc2b..5116bac 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -81,11 +81,10 @@ pub async fn search( .insert_header(("location", "/")) .finish()) } else { - // Initialize the page url as an empty string - let mut page_url = String::new(); + let page_url: String; // Declare the page_url variable without initializing it - // Find whether the page is valid page number if not then return - // the first page number and also construct the page_url accordingly + // ... + let page = match params.page { Some(page_number) => { if page_number <= 1 { @@ -99,7 +98,7 @@ pub async fn search( "http://{}:{}/search?q={}&page={}", config.binding_ip_addr, config.port, query, page_number ); - + page_number } } @@ -111,11 +110,13 @@ pub async fn search( req.uri(), 1 ); - + 1 } }; - + + // Use the page_url variable as needed + // fetch the cached results json. let cached_results_json = redis_cache.cached_results_json(&page_url); // check if fetched results was indeed fetched or it was an error and if so From ae2fdb557cd801a833e2e2c85b1dea14edd5e7c4 Mon Sep 17 00:00:00 2001 From: MD AL AMIN TALUKDAR <129589283+alamin655@users.noreply.github.com> Date: Sat, 20 May 2023 08:01:24 +0530 Subject: [PATCH 007/392] Removed unnecessary comment --- src/server/routes.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/server/routes.rs b/src/server/routes.rs index 5116bac..ed2299f 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -114,9 +114,7 @@ pub async fn search( 1 } }; - - // Use the page_url variable as needed - + // fetch the cached results json. let cached_results_json = redis_cache.cached_results_json(&page_url); // check if fetched results was indeed fetched or it was an error and if so From 6e19246cb2a02f06d6e2ca3851d78b2f01f7bdb0 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 22 May 2023 13:07:37 +0300 Subject: [PATCH 008/392] fix: reduce the time to build user-agents --- Cargo.lock | 1 + Cargo.toml | 1 + src/search_results_handler/user_agent.rs | 20 +++++++++++--------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85310e9..cec7cf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3328,6 +3328,7 @@ dependencies = [ "handlebars", "log", "md5", + "once_cell", "rand 0.8.5", "redis", "reqwest 0.11.17", diff --git a/Cargo.toml b/Cargo.toml index ce99ca3..efe9435 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,3 +21,4 @@ rlua = {version="*"} redis = {version="*"} md5 = {version="*"} rand={version="*"} +once_cell = {version="*"} diff --git a/src/search_results_handler/user_agent.rs b/src/search_results_handler/user_agent.rs index 09dd684..13166bf 100644 --- a/src/search_results_handler/user_agent.rs +++ b/src/search_results_handler/user_agent.rs @@ -1,13 +1,8 @@ //! This module provides the functionality to generate random user agent string. -use fake_useragent::{Browsers, UserAgentsBuilder}; +use fake_useragent::{Browsers, UserAgents, UserAgentsBuilder}; -/// A function to generate random user agent to improve privacy of the user. -/// -/// # Returns -/// -/// A randomly generated user agent string. -pub fn random_user_agent() -> String { +static USER_AGENTS: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| { UserAgentsBuilder::new() .cache(false) .dir("/tmp") @@ -21,6 +16,13 @@ pub fn random_user_agent() -> String { .set_mozilla(), ) .build() - .random() - .to_string() +}); + +/// A function to generate random user agent to improve privacy of the user. +/// +/// # Returns +/// +/// A randomly generated user agent string. +pub fn random_user_agent() -> String { + USER_AGENTS.random().to_string() } From ab20f08e42251dbb052ec5aa08534187d8283f03 Mon Sep 17 00:00:00 2001 From: gotoworq <110043355+gotoworq@users.noreply.github.com> Date: Mon, 22 May 2023 14:44:46 -0400 Subject: [PATCH 009/392] Added logo, and changed logon to be on main page --- public/images/websurfx_logo.png | Bin 0 -> 8264 bytes public/templates/index.html | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 public/images/websurfx_logo.png diff --git a/public/images/websurfx_logo.png b/public/images/websurfx_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..9449e33ff4400ee97da4186599ead26e0ead74a7 GIT binary patch literal 8264 zcmV-OAh+L%P)FK#oU~nHKzykz8ilj(NCUjet zWyZ=T<=9wxH^RH#-CzE(5&Oe`ynh_AVTWVoSRZoO(n^#p%aV6RLJ}zwgg}A>MS!?5 z1I*oXcK7uCj;h=r-P6+008JhnIcap_OBOZ`=$_Qq;6@IdsEtC6fWo4qQNUz<(a9IW8? zuTHSGP>WW-&igv<0!#TkwMhLs?g_TP;V)Ob=BQsc`S%d3X8(3T%~7AK{#nQUMU6cN zRn@II>epkM_ZAO;;V(3!=BU3iw#fi(9#+?Zwb{)jTGtl!>n4A*v5mUDwy3`{w$adR zCbpm9ck5b1)UTWTt%2>(`3N;d{gtsD24%}&`x*WM7F4c5>epaz*RcgrE7V_^@AW5Q z!?9$SZi~LI73$ZqWl=NFL4~5VM*SuFxn{b5+o0xq9BYF5HQ&v(i|yI<2vB=gzmBbl zntVW%%~qfQR5WGBBgSK2Qo0U`se$bIA2@eDB*#KU13+>tfV#J{Il!e71(&>08$d3LXb-N{0oEYO0wkL;|N9k#EZv97)eOabI#ZUmrC{Y}G?45yVvwiHVB z#eo(z=Vc?quFZGLtes2~em0Rz__MLaCOU z?eGZ48NW`6In+I5o6<}%-S6yuQakz?-qfq^J^@75z`Fo--q*3g*lg+pkePPObn&Ca z8JU^`DU2t77rhNKmlm@lo2XtLuua|l&31UIfozkKb7=T^Lz{i`SN71a0qR!2jtvEF z0jpmXA~9tTf5iMv&h+3iVw=wUelL1T;=?`#cY46b5dr>YOq9!3Rkmhr<|@$d#ZQL$ zPEX+Q+J8*j_mp+x9p}!6B%HQ~h$wn39cAnYGXkWaw4W5BT@2AIruXQPpH82(7TulD`xR^HSm2Ty~?N){} z^3R8yh`IR<-W-QVY3?(Q|F(YMlS8Oy;N zQR>p6!(hBbo2X5{ic?msaHwkuRxi0ZSiN$4efVGd)q{S<5uz6T*^o%u^vAoDXPTJC zpjpE`KD>~itXR3Kdx}lqJt`XS*r)D34w_eK*~tRUO4k8r^s;^Zj1#$$U%g_1W)|v{ z(J>It{nzkr@rx)O3{bPSNDld|$j5mpMTv8rd!{7GI2e zeQd8+T(H63)V6L0^?f1Uxes+e01yBg0k!)uZ|+lf9hcEjXXImNdH{_e^EX+y&fS0Y z)_{xv$8}26ny3#o$^H`jMhkD#vDg_t;%9loqhAJa_GYo&Re>Z{p%okER_m^HF zKs18f2=eA$rR$J1QvgU-hPUr#O+6qTnVqN9Tu~lX1B0!ANYODTa`V2opbBI;U1?pX znbt7-5Bkwt^3YG3WJ$;faLOALcPg*ke{6=;H+)Ll?&|lLf`MipXa)dqOm^~X001OO z*pN@-S%6H<*_VH2#UuAx+f_mmmHNYoF6!T1fTt$ezsrk_YFRQcLqve{ae1r(pJ?P7 z0|2-%PdC$KZFZ#Rep4U0ibfDr&&r~T&KldRTsPC!&9mmOexzg~MJua0&d)$*2d1VqRR__I5r-y?VXaNMMUG?|;td)Nf4uc*w|xaQK7 znKv)JW`6L`Vtx|k7U(|!03d)0RS=M^$omrMYV6-~hJQtcbmra(Ma3((vvfPVg<5pq zsgDfU8fSoHrmSmk=FWd#&QEWwQtO9wjcH-7xBA%T7GlL3L4XyF*vi{M?mP8~K(1+G zFC8&`+aCVNIQ~1<#h*G;%dPTm2>LwOZ9t2LI)k`P18Ew~fmMy51V9xIPu|x@005mq zmyY}W7*Jr+r0Xe~wV+BDl(WB4g*{%@Wq{5A0EV)3B}pk!xYeRzBv<(8#)<;P1}X>> z=5itSyz&V_u?3y?`EZ{PJsgI!bU9h}6_P4V0K=`jT5-RrPmU?pG&iNpymRM*(s@8X z`dOv(fIax0edP@i9oujX8x(kaPB)XBQ^DkOiUTY06s@b_?* zt|Y~g2JT2my5c*19{ll?e0^3V9eBElz0}G545qBb*>Il+|6-`nq7A`)xB9}NRCEpH zWG-b7p7(y?k5Tul-G_Knx7L5mzVm@~4Fa9>lM8@f{unfXgZR)|kAVApl5p zz>6)c&x8M=PwAZ&|1wQ8CH9|O4i5r&8GN;!c^Q3cOn4Z4I*hw~g%#hYL)d5VzZw}7{XY}pwHc8pu@r~n^!we);XtJ;^vx0Hmvb^d{XG+v=i~ClX7>43-l8J_ zjAZG54Lg^T8|f5oLLvid9&P{A`6aE@LOOD4z`F5{=jm4f86cwO;SFuPWvANrFmLS# zM98JcUc!*B4$n8?zv<#V1_FR7;nj%zI9YIHauRlXF~45B$H!tejpnGh7#URHkw)HH zvYML^5LWU1dHUHL4Q1g@7H(vy*Mmiw1{FgJj^-d>)Ug_rheFd2-DL5CW=z@SC zxRIh39lH&Tm~=G-^G@OVSdRLAti9asz8j~}+#**dOlZ@g$Ael)*mW$oSWi)`&x1W4 zEX_(9&rz!mZ^tNV(@UMoUf<%qq(g)0LJi#>I2>RNB_*#WX&?io@tkQ3<}B**uuyq0 zXfRDf*+oe~FqEPF0oHBc<)r-IqvB$c#D-3XHX*K0Rz^f_vS8cYl#yiTcvCm9MTH3v zeIdmcVl6wB-6tp&mD9JK$(zpT<-&%>yFrfu&xCpB0(%D2a?)Jv>wyf-SWsMtdpw9& zQ+PAII209Dzg5F`Vr8Q~nuB*@)S_XVR=DzL171zTv<2w}cll@#JG8}0ETW3QkxB4& zRPHcfxS1!@B8GFckckNg18j!} ziq_3Nv7$hj>dchPR% zp=`MvR%5`lMalAs@3={$CgdbYS0HN6rp!W}?z>eVEdY3ZR(_O_rQH)$@r5w&FLAUE z_z?kqG%0SR?n&@#-PA|MymuF;C&#pJe<0^4@8(N53%1Dk)N<8Y8S}!Aa_@a#CTH(T z9{}k0V4s&6C7qEGWK!9n2`U@c4t;S#NiOCk1lTSzj+-!7o}JzA!BQ{9$H6EeY*587 zw&FpTTRfDN^A1%QRQKcVexX;5z_<;wTP!8OmH+^Q85qe@k+QQET}V*GT(TfvBEpk! z;LRuvX3E}bM;q9aO$A3NsKAi`yPA}jl5~$!9@gjZATm!td+1Y2=K-0V6;lH;Ixb=p zG812}6p@Hcq<{D)O3y>Qt)KbBtf`wdR9m=RECHzdFxVpJC#y>e%ewTs`N5AWlAl+J zJ`Xf&Wmh65R9I+9Wf9D71DjNe*#Lzlt(6iP01^(QoiZ_4Wy)T5>c#eKKM=v(Blz_AG%3#_CVtv#nA~R#(IwvM? zseO;=M?Z(YMlv&$iHqo%n7k#X21G1EcCNCZ6!zFov2ySKaBaK?fS!^GXdc}L9`cu6;RqQCO+|gmU*)S|9stB` z$cjbh7yud+^l->Z0)p#lc+7;R@`rGrAOG_X{!f$QttjQ6keg|`=?V?4#^E8Vvi{)a zrdzDEElCH?MCnYliYZ>cpc`*SWv78JH1neC7*O!p5bN_WQ__2JIc?svj|+e2TaB1Au}+wn+o+b`1ruB8FO&00`MMp(PZzJTNgpPe=Wo0f&R+dsx5giqi17d1`9E&%0IFXwm91)ukv51%* zv2VP?TXyJ2PAT1oQ8V<%zL2nHSc@Udd-zmQk_bTa zqUPoPFd~p}6zk!}u6K7*(HLc3wspji57p2`|3;|t_mK)Uzu2jkkxRx?zssAt^8MnBeJmok zSRl+oUA(D_c>-hv3e^eQc5#O$y$cG4&aC%MxfK6~VMp0Y5)WIKi zO}O{?@MsV}PA+=GZhpA^B?-8yxV}qlmA<}vy1$x&eFhv2;(Kv%F(F^-REq3Z5uOgS zNt1pV-N=m@MFrPIeWm|#{qUzmJYtVr0+LE=zwwEemFB);?c5*c4XrXUoBP$bGe7=Q z&-35VcAsRy7IoJX=Ea|Sp8dDHp`9EvZ!9P=V-J61U;Cvr(@giVP$z5ZX8y3+d4S3u zgP1clAf^VzgbO{F%$S&ivD)g`R#W8iKfa(Oi0DB3dMkrbU7r< zh#Fx!;2bq>Uc85Zf9s@bpvr%SQqv6 zLocAWL0o$a91{T4T~F{(M^Q)X^3R=-i`dYr?md}1_g!adFmw7Zfk@qRQakiKS=rp# zZ+o8q10L!i0w6)b7S`NLW(K4~!lrc0nYd2#Gcp}TRmV^V4>a@E-K=ew(shu`w9F+b z6)n69CU05S-ekic(V8{`jGHi$rGq8LwL{0im_!@$V}{iaQdm{gkhP&KY! zC7*sZV$1kDC6d((Eu$NqG~wT-yz2Y05Lb3IsKPjDb&4B=|@hPmtIf*_#xS_OMvp^j};aKPBgMlxA3!3S#0k|32N8G%iW4r;`W?qz=<3?J-fmCQP)L% zR1E^ahF0JTI2L8%ARXX^eHo~zWar4t$dUwo5gp6?{F}aDtJ<+ofA|#YzRb_RsUQB7 zcIY|aCCiv4Pl9#nG+9}4%9ezk$W7F}sOU^HK=Y#RV-0OA(4zJq)Al|InpZ?_IFq-; z)Bu_36(56y1J}~@M2@;ja?RUx{6f1z;Jg*3qRH9i#oy^*O)9<`S+qmsiY;QLfhPcH z(V(~FgwCWxS5kBl$M>Kuae3F?{^r&d*HvQFd%*%mIGwxAvToHH{_xynNmM070s`k%v# zf24Hm)epVEL!IWwr=5vw`Xis`tvdnBlVY8mJ@9VfNVZ^$(!Pf^v@>r57y;%BF;9?$ zMY%NCIhl^q{G>fM0#+8d3XC(u&-XnIKq5W~X3CivTIMXb%Sn2DM*dz`nPu(L;Tyfm z7ux80nk+%Pz1ZhvwxrXu@?xTF7*>W%O}djQdp5!v>-JE7s|^4g^x-2xHlCCH-o+Mb3=CEo;b~N$B4aeDVvB~? zS6W{6a%d{g2Wiux-^<2b2QcjQ;aA((em_e(WK_Nh>D)Z+^_S^+IKWP|(BF)1oZGX` zUsd7WMz#A8^EZ)|6_FbRMIxIfg{lmB{9Rq{o9v>d?FAkL{;Z}yOl`OmN^x$V(*sg`)yq=P;PRfyT zGuG#2#~WElMT5ijG-MX0EegVZAAX{#kn56i=uA}p-x(^M)|e6k3LXwDDxc5GAIy-- z;HeOMypcsrQW=)+qVjQcBwZ^x9i-yBied`FP=*pq&!iN@{eFDfU68?0MwS-b@5h(B z`9nbl06IgC5HV?{B*>N)k#*a01*|PH}|Sthsd#HJW|FvBAJ?#)34B4%^7@OX67N6mg$(Bk04V(3LfrZfoAkHFeAWpFQ|r$O~~0HF@DvF z+@MU1g?qIF&!VC^lQ(5LMs^My3xq?$UMBQuNf^q~aF*Kj$~a6Oa~A#cl>FX=tgt;g z!#!R+7+`Z2y&IQr$LMO3nlV?0N%PKf_ELlrtW1PO;c9Ct0O185BQ`q*!_@h`S= zAIE>4mbbHTC{RXyhS2R{#~Yb~Fp{N{jrgDXlxM?SFP{N0gl+?$4YL!C*rwx9w(KAs zgTcv0{G%PpCzmexcmoToFl)of5c|VkrKiO4YtisTi1ipaXF=44u!f&);h$|`db!J} zG2H289Xi~}E^h8w57zfHa>%qjC%rHJM;_`D@rZTh4fDgFut2lt>8~q0juw1(5;8t1 zrU%W-zi@_rEi>^&pOi5kY*Ts;D?JaP<`J{Q_TYIjeW!TlIP*3ze=~3G=bmFF=b*#%-pf>T%c4G48IcYVWCcY@VuBDsB(_^($E>~^x{!Jc6+cv z0RTt|xRRvv^U}Qn000dN9&dn<3a`ylakC#s=r!QU5ZmF!JlWAje|o(7YkRf9)^ zXmFf1DN$a@yh9Tvgq6bA<*$t93JnP>(6VqnEodTFvIt1P5yEPrpebnDDro5A&}pE) z;Q6H;IcSmy~=g~-D?e?bvdUEOuu^QnKY zH1)_#!WsT`?)(qPOsl(&YezoA!`%hnUDz@)Ez?nF{F~xfh4LrdK-~( z4o0u=eTXwQw3m|tUZ&>gDP^M$ZRbYxt&Q{quP{%5;3f?a#wWY9;)CT}yj5o9? zJ08^!e@f{&OlDTh4B0o&+Cv|LbXcgHhkJNyKWpw)T6cm{WhY7zGBz&fr(`NBXNH`) z5z5Y!nE~4*VUd+3;efEnu__{Y2!Qz~N7YdCq3%O(18eT(z9wh#mVM^~aBM(Eb`C1H zde^Zs>eOEWsCjv?mACI!_ncJvjsT!c%{gONoS|Qf$yOITAMI(qyJd*k!9JOGJsQTxN_J0rWJWs)^`0Sy@0<=l$BSZM+>Q zDoXd@*V0kZSfH8tLcHsM(!Li%9ZdI;m6h>`n7$)3amuC0Op%nRXn+iaMfs?UnE~C4 zB+1H&xlyt*rL|W4U-_lWD8v_9AxTn-F0(Bqr22uB;?&aZLU>rR`;3_y4 zI94eaMRJyM2{5zyKqN_r%#7=3&NbQoI@XOk^*0@L>Td#;V|49$I{@H1$Y?8L<^HXi z5!eQ({l2at>X)GQ`@JTxy}P-#s9!hyTLZP})U4Hqn(p?kjt8M@Ynl2><3Sj?&A~P>`BeBesQssA)PckE%^$ z%iP=Ys2l$Kh3#YV7x6$?{cSQ<_Yx0;t5dGK`y}ex!m14aN;e;LZ12vYB|fnwBx(qC z_OA;obj!9e`7Ufbho{c|yTJp*d-utZg;RBF8+Xxwns4jI;cj?et60k0feNc%%Z**F z+p~)M4+QR#_uH_~wv)qCXa8z(m(8#J-dadmD+Mdup%R%IMxFgD<1TkR)Nns7uljbf z`YzPj@4|z_`xQ|_{cY0BWhOQ&N^f&KP&?epk4mZ1u>TMGr4mw5JtNEj0000header this}}
- Websurfx meta-search engine logo + Websurfx meta-search engine logo {{>search_bar}}
From ff325153f06dbcb18c78a30dbf6c076996f75d7a Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Tue, 23 May 2023 00:31:46 +0000 Subject: [PATCH 010/392] add format and clippy checks to the CI jobs --- .github/workflows/rust_format.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust_format.yml b/.github/workflows/rust_format.yml index d865c8c..c300863 100644 --- a/.github/workflows/rust_format.yml +++ b/.github/workflows/rust_format.yml @@ -19,7 +19,16 @@ jobs: profile: minimal toolchain: stable components: rustfmt, clippy - + - name: Format + uses: actions-rs/cargo@v1 + with: + command: fmt + args: -- --check + - name: Clippy + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --all-features --all-targets --all - name: Run cargo check uses: actions-rs/cargo@v1 with: From cecffe41555d76ed254ead5a7cebf277e6b98172 Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Tue, 23 May 2023 09:34:46 +0000 Subject: [PATCH 011/392] make format happy --- src/cache/mod.rs | 2 +- src/search_results_handler/aggregation_models.rs | 2 +- src/server/routes.rs | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 91a91ca..de7dd4e 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -1 +1 @@ -pub mod cacher; +pub mod cacher; diff --git a/src/search_results_handler/aggregation_models.rs b/src/search_results_handler/aggregation_models.rs index 4fe670e..b6e6b81 100644 --- a/src/search_results_handler/aggregation_models.rs +++ b/src/search_results_handler/aggregation_models.rs @@ -116,7 +116,7 @@ impl RawSearchResult { } } -/// A named struct to store, serialize, deserialize the all the search results scraped and +/// A named struct to store, serialize, deserialize the all the search results scraped and /// aggregated from the upstream search engines. /// /// # Fields diff --git a/src/server/routes.rs b/src/server/routes.rs index ed2299f..85c522d 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -81,10 +81,10 @@ pub async fn search( .insert_header(("location", "/")) .finish()) } else { - let page_url: String; // Declare the page_url variable without initializing it + let page_url: String; // Declare the page_url variable without initializing it // ... - + let page = match params.page { Some(page_number) => { if page_number <= 1 { @@ -98,7 +98,7 @@ pub async fn search( "http://{}:{}/search?q={}&page={}", config.binding_ip_addr, config.port, query, page_number ); - + page_number } } @@ -110,11 +110,11 @@ pub async fn search( req.uri(), 1 ); - + 1 } }; - + // fetch the cached results json. let cached_results_json = redis_cache.cached_results_json(&page_url); // check if fetched results was indeed fetched or it was an error and if so From 05272884bac1e6068f58180e964227126b9f9067 Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Mon, 22 May 2023 01:13:06 +0000 Subject: [PATCH 012/392] supports the option to add a random delay --- src/config_parser/parser.rs | 13 +++++++++++++ src/search_results_handler/aggregator.rs | 10 +++++++--- src/server/routes.rs | 2 +- websurfx/config.lua | 5 +++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index 4625bd8..21a9bf5 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -20,6 +20,14 @@ pub struct Config { pub binding_ip_addr: String, pub style: Style, pub redis_connection_url: String, + pub aggregator: AggreatorConfig, +} + +/// Configuration options for the aggregator. +#[derive(Clone)] +pub struct AggreatorConfig { + /// Whether to introduce a random delay before sending the request to the search engine. + pub random_delay: bool, } impl Config { @@ -41,6 +49,8 @@ impl Config { .load(&fs::read_to_string("./websurfx/config.lua")?) .exec()?; + let aggregator_config = globals.get::<_, rlua::Table>("aggregator")?; + Ok(Config { port: globals.get::<_, u16>("port")?, binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?, @@ -49,6 +59,9 @@ impl Config { globals.get::<_, String>("colorscheme")?, ), redis_connection_url: globals.get::<_, String>("redis_connection_url")?, + aggregator: AggreatorConfig { + random_delay: aggregator_config.get::<_, bool>("random_delay")?, + }, }) }) } diff --git a/src/search_results_handler/aggregator.rs b/src/search_results_handler/aggregator.rs index 5133094..8b86972 100644 --- a/src/search_results_handler/aggregator.rs +++ b/src/search_results_handler/aggregator.rs @@ -29,6 +29,7 @@ use crate::engines::{duckduckgo, searx}; /// /// * `query` - Accepts a string to query with the above upstream search engines. /// * `page` - Accepts an u32 page number. +/// * `random_delay` - Accepts a boolean value to add a random delay before making the request. /// /// # Error /// @@ -38,14 +39,17 @@ use crate::engines::{duckduckgo, searx}; pub async fn aggregate( query: &str, page: u32, + random_delay: bool, ) -> Result> { let user_agent: String = random_user_agent(); let mut result_map: HashMap = HashMap::new(); // Add a random delay before making the request. - let mut rng = rand::thread_rng(); - let delay_secs = rng.gen_range(1..10); - std::thread::sleep(Duration::from_secs(delay_secs)); + if random_delay { + let mut rng = rand::thread_rng(); + let delay_secs = rng.gen_range(1..10); + std::thread::sleep(Duration::from_secs(delay_secs)); + } // fetch results from upstream search engines simultaneously/concurrently. let (ddg_map_results, searx_map_results) = join!( diff --git a/src/server/routes.rs b/src/server/routes.rs index 85c522d..0f84cc9 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -127,7 +127,7 @@ pub async fn search( } Err(_) => { let mut results_json: crate::search_results_handler::aggregation_models::SearchResults = - aggregate(query, page).await?; + aggregate(query, page, config.aggregator.random_delay).await?; results_json.add_style(config.style.clone()); redis_cache .cache_results(serde_json::to_string(&results_json)?, &page_url)?; diff --git a/websurfx/config.lua b/websurfx/config.lua index 916a9b3..7dfd515 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -19,3 +19,8 @@ theme = "simple" -- the theme name which should be used for the website -- Caching redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. + +-- Aggregator +aggregator = { + random_delay = false, -- whether to add random delay before sending the request to the search engine +} \ No newline at end of file From 9773cee38dd312696bd17168e5758176a5563b8e Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Tue, 23 May 2023 00:56:44 +0000 Subject: [PATCH 013/392] change the 'aggregator' option to 'production_use' --- src/config_parser/parser.rs | 15 +++++++++++---- websurfx/config.lua | 7 +++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index 21a9bf5..4b73a73 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -49,7 +49,16 @@ impl Config { .load(&fs::read_to_string("./websurfx/config.lua")?) .exec()?; - let aggregator_config = globals.get::<_, rlua::Table>("aggregator")?; + let production_use = globals.get::<_, bool>("production_use")?; + let aggregator_config = if production_use { + AggreatorConfig { + random_delay: true, + } + } else { + AggreatorConfig { + random_delay: false, + } + }; Ok(Config { port: globals.get::<_, u16>("port")?, @@ -59,9 +68,7 @@ impl Config { globals.get::<_, String>("colorscheme")?, ), redis_connection_url: globals.get::<_, String>("redis_connection_url")?, - aggregator: AggreatorConfig { - random_delay: aggregator_config.get::<_, bool>("random_delay")?, - }, + aggregator: aggregator_config }) }) } diff --git a/websurfx/config.lua b/websurfx/config.lua index 7dfd515..1c0be7d 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -20,7 +20,6 @@ theme = "simple" -- the theme name which should be used for the website -- Caching redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. --- Aggregator -aggregator = { - random_delay = false, -- whether to add random delay before sending the request to the search engine -} \ No newline at end of file +production_use = false -- whether to use production mode or not +-- if production_use is set to true + -- there will be a random delay before sending the request to the search engines, this is to prevent the search engines from blocking the ip address \ No newline at end of file From 4b70a74ff6196a00d8878333ab4b56b065427230 Mon Sep 17 00:00:00 2001 From: zhou fan <1247714429@qq.com> Date: Tue, 23 May 2023 17:30:36 +0800 Subject: [PATCH 014/392] Update websurfx/config.lua Co-authored-by: neon_arch --- websurfx/config.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/websurfx/config.lua b/websurfx/config.lua index 1c0be7d..c30f376 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -20,6 +20,6 @@ theme = "simple" -- the theme name which should be used for the website -- Caching redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. -production_use = false -- whether to use production mode or not +production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) -- if production_use is set to true - -- there will be a random delay before sending the request to the search engines, this is to prevent the search engines from blocking the ip address \ No newline at end of file + -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. \ No newline at end of file From ea013e718edae23136befa9609100960431ab5f4 Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Tue, 23 May 2023 10:00:35 +0000 Subject: [PATCH 015/392] make format happy --- src/config_parser/parser.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index 4b73a73..f8dac14 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -51,9 +51,7 @@ impl Config { let production_use = globals.get::<_, bool>("production_use")?; let aggregator_config = if production_use { - AggreatorConfig { - random_delay: true, - } + AggreatorConfig { random_delay: true } } else { AggreatorConfig { random_delay: false, @@ -68,7 +66,7 @@ impl Config { globals.get::<_, String>("colorscheme")?, ), redis_connection_url: globals.get::<_, String>("redis_connection_url")?, - aggregator: aggregator_config + aggregator: aggregator_config, }) }) } From 89796a054f0b0b9535e40d13e4a4effc242737e6 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 23 May 2023 23:47:36 +0300 Subject: [PATCH 016/392] feat: add ability to put config file on different paths --- src/config_parser/parser.rs | 44 ++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index 4625bd8..66a9284 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -3,7 +3,7 @@ use super::parser_models::Style; use rlua::Lua; -use std::fs; +use std::{format, fs, path::Path}; /// A named struct which stores the parsed config file options. /// @@ -32,13 +32,13 @@ impl Config { /// or io error if the config.lua file doesn't exists otherwise it returns a newly contructed /// Config struct with all the parsed config options from the parsed config file. pub fn parse() -> Result> { - let lua = Lua::new(); - - lua.context(|context| { + Lua::new().context(|context| -> Result> { let globals = context.globals(); context - .load(&fs::read_to_string("./websurfx/config.lua")?) + .load(&fs::read_to_string( + Config::handle_different_config_file_path()?, + )?) .exec()?; Ok(Config { @@ -52,4 +52,38 @@ impl Config { }) }) } + /// A helper function which returns an appropriate config file path checking if the config + /// file exists on that path. + /// + /// # Error + /// + /// Returns a `config file not found!!` error if the config file is not present under following + /// paths which are: + /// 1. `~/.config/websurfx/` if it not present here then it fallbacks to the next one (2) + /// 2. `/etc/xdg/websurfx/config.lua` if it is not present here then it fallbacks to the next + /// one (3). + /// 3. `websurfx/` (under project folder ( or codebase in other words)) if it is not present + /// here then it returns an error as mentioned above. + fn handle_different_config_file_path() -> Result> { + if Path::new( + format!( + "{}/.config/websurfx/config.lua", + std::env::var("HOME").unwrap() + ) + .as_str(), + ) + .exists() + { + Ok(format!( + "{}/.config/websurfx/config.lua", + std::env::var("HOME").unwrap() + )) + } else if Path::new("/etc/xdg/websurfx/config.lua").exists() { + Ok("/etc/xdg/websurfx/config.lua".to_string()) + } else if Path::new("./websurfx/config.lua").exists() { + Ok("./websurfx/config.lua".to_string()) + } else { + Err(format!("Config file not found!!").into()) + } + } } From c5b62f1087781b0e98dcbe03d4ca155d08ffedf9 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 24 May 2023 10:50:08 +0300 Subject: [PATCH 017/392] chore: Bump version to v0.8.0 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index efe9435..d19acb2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.6.0" +version = "0.8.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From ea3f21139f3fcd25d2915838ffc83b5bf4ea3566 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 24 May 2023 11:53:40 +0300 Subject: [PATCH 018/392] chore: fix the lock error during build --- Cargo.lock | 194 ++++++++++++++++++++++++++--------------------------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cec7cf7..02bf14f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,19 +4,19 @@ version = 3 [[package]] name = "actix-codec" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" +checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ "bitflags", "bytes 1.4.0", "futures-core", "futures-sink", - "log", "memchr", "pin-project-lite", - "tokio 1.28.0", + "tokio 1.28.1", "tokio-util", + "tracing", ] [[package]] @@ -53,7 +53,7 @@ dependencies = [ "actix-service", "actix-utils", "ahash 0.8.3", - "base64 0.21.0", + "base64 0.21.1", "bitflags", "brotli", "bytes 1.4.0", @@ -62,7 +62,7 @@ dependencies = [ "encoding_rs", "flate2", "futures-core", - "h2 0.3.18", + "h2 0.3.19", "http 0.2.9", "httparse", "httpdate", @@ -75,7 +75,7 @@ dependencies = [ "rand 0.8.5", "sha1", "smallvec 1.10.0", - "tokio 1.28.0", + "tokio 1.28.1", "tokio-util", "tracing", "zstd", @@ -111,7 +111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" dependencies = [ "futures-core", - "tokio 1.28.0", + "tokio 1.28.1", ] [[package]] @@ -128,7 +128,7 @@ dependencies = [ "mio 0.8.6", "num_cpus", "socket2", - "tokio 1.28.0", + "tokio 1.28.1", "tracing", ] @@ -201,7 +201,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" dependencies = [ "actix-router", - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", "syn 1.0.109", ] @@ -315,9 +315,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "3f1e31e207a6b8fb791a38ea3105e6cb541f55e4d029902d3039a4ad07cc4105" [[package]] name = "bit-set" @@ -381,9 +381,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -605,7 +605,7 @@ dependencies = [ "itoa 1.0.6", "matches", "phf 0.10.1", - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", "smallvec 1.10.0", "syn 1.0.109", @@ -628,7 +628,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", "rustc_version 0.4.0", "syn 1.0.109", @@ -636,9 +636,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -730,7 +730,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", "syn 1.0.109", "synstructure", @@ -959,9 +959,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ "bytes 1.4.0", "fnv", @@ -971,16 +971,16 @@ dependencies = [ "http 0.2.9", "indexmap", "slab", - "tokio 1.28.0", + "tokio 1.28.1", "tokio-util", "tracing", ] [[package]] name = "handlebars" -version = "4.3.6" +version = "4.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "035ef95d03713f2c347a72547b7cd38cbc9af7cd51e6099fb62d586d4a6dee3a" +checksum = "83c3372087601b532857d332f5957cbae686da52bb7810bf038c3e3c3cc2fa0d" dependencies = [ "log", "pest", @@ -1035,7 +1035,7 @@ dependencies = [ "log", "mac", "markup5ever 0.11.0", - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", "syn 1.0.109", ] @@ -1149,7 +1149,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.18", + "h2 0.3.19", "http 0.2.9", "http-body 0.4.5", "httparse", @@ -1157,7 +1157,7 @@ dependencies = [ "itoa 1.0.6", "pin-project-lite", "socket2", - "tokio 1.28.0", + "tokio 1.28.1", "tower-service", "tracing", "want 0.3.0", @@ -1185,7 +1185,7 @@ dependencies = [ "bytes 1.4.0", "hyper 0.14.26", "native-tls", - "tokio 1.28.0", + "tokio 1.28.1", "tokio-native-tls", ] @@ -1301,9 +1301,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.62" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68c16e1bfd491478ab155fd8b4896b86f9ede344949b641e61501e07c2b8b4d5" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] @@ -1338,9 +1338,9 @@ checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "linux-raw-sys" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "local-channel" @@ -1631,9 +1631,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -1749,9 +1749,9 @@ checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -1863,7 +1863,7 @@ dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", "proc-macro-hack", - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", "syn 1.0.109", ] @@ -1942,9 +1942,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" dependencies = [ "unicode-ident", ] @@ -1974,7 +1974,7 @@ version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", ] [[package]] @@ -2213,9 +2213,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "d1a59b5d8e97dee33696bf13c5ba8ab85341c002922fba050069326b9c498974" dependencies = [ "aho-corasick", "memchr", @@ -2224,9 +2224,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "reqwest" @@ -2264,16 +2264,16 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.17" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.0", + "base64 0.21.1", "bytes 1.4.0", "encoding_rs", "futures-core", "futures-util", - "h2 0.3.18", + "h2 0.3.19", "http 0.2.9", "http-body 0.4.5", "hyper 0.14.26", @@ -2289,7 +2289,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.1", - "tokio 1.28.0", + "tokio 1.28.1", "tokio-native-tls", "tower-service", "url 2.3.1", @@ -2410,9 +2410,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ "bitflags", "core-foundation", @@ -2423,9 +2423,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" dependencies = [ "core-foundation-sys", "libc", @@ -2482,22 +2482,22 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.162" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b2f6e1ab5c2b98c05f0f35b236b22e8df7ead6ffbf51d7808da7f8817e7ab6" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.162" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2a0814352fd64b58489904a44ea8d90cb1a91dcb6b4f5ebabc32c8318e93cb6" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -2680,7 +2680,7 @@ checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" dependencies = [ "phf_generator 0.7.24", "phf_shared 0.7.24", - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", "string_cache_shared", ] @@ -2693,7 +2693,7 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", ] @@ -2720,18 +2720,18 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.15" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", "unicode-ident", ] @@ -2742,7 +2742,7 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", "syn 1.0.109", "unicode-xid 0.2.4", @@ -2796,9 +2796,9 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -2875,9 +2875,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.0" +version = "1.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" dependencies = [ "autocfg 1.1.0", "bytes 1.4.0", @@ -2940,9 +2940,9 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -2952,7 +2952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.28.0", + "tokio 1.28.1", ] [[package]] @@ -3037,7 +3037,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.28.0", + "tokio 1.28.1", "tracing", ] @@ -3061,9 +3061,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", ] @@ -3243,9 +3243,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b6cb788c4e39112fbe1822277ef6fb3c55cd86b95cb3d3c4c1c9597e4ac74b4" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3253,24 +3253,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e522ed4105a9d626d885b35d62501b30d9666283a5c8be12c14a8bdafe7822" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", - "syn 2.0.15", + "syn 2.0.16", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.35" +version = "0.4.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "083abe15c5d88556b77bdf7aef403625be9e327ad37c62c4e4129af740168163" +checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3280,9 +3280,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "358a79a0cb89d21db8120cbfb91392335913e4890665b1a7981d9e956903b434" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ "quote 1.0.27", "wasm-bindgen-macro-support", @@ -3290,28 +3290,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4783ce29f09b9d93134d41297aded3a712b7b979e9c6f28c32cb88c973a94869" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.58", "quote 1.0.27", - "syn 2.0.15", + "syn 2.0.16", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a901d592cafaa4d711bc324edfaff879ac700b19c3dfd60058d2b445be2691eb" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "web-sys" -version = "0.3.62" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b5f940c7edfdc6d12126d98c9ef4d1b3d470011c47c76a6581df47ad9ba721" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" dependencies = [ "js-sys", "wasm-bindgen", @@ -3319,7 +3319,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.6.0" +version = "0.8.0" dependencies = [ "actix-files", "actix-web", @@ -3331,12 +3331,12 @@ dependencies = [ "once_cell", "rand 0.8.5", "redis", - "reqwest 0.11.17", + "reqwest 0.11.18", "rlua", "scraper", "serde", "serde_json", - "tokio 1.28.0", + "tokio 1.28.1", ] [[package]] From a7a28ed8c64d54c7f47a53106032d49c2aeef058 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 24 May 2023 12:01:36 +0300 Subject: [PATCH 019/392] chore: make websurfx directory and config file names as constants --- src/config_parser/parser.rs | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index 66a9284..bbeba86 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -5,6 +5,10 @@ use super::parser_models::Style; use rlua::Lua; use std::{format, fs, path::Path}; +// ------- Constants -------- +static COMMON_DIRECTORY_NAME: &str = "websurfx"; +static CONFIG_FILE_NAME: &str = "config.lua"; + /// A named struct which stores the parsed config file options. /// /// # Fields @@ -67,20 +71,29 @@ impl Config { fn handle_different_config_file_path() -> Result> { if Path::new( format!( - "{}/.config/websurfx/config.lua", - std::env::var("HOME").unwrap() + "{}/.config/{}/config.lua", + std::env::var("HOME").unwrap(), + COMMON_DIRECTORY_NAME ) .as_str(), ) .exists() { Ok(format!( - "{}/.config/websurfx/config.lua", - std::env::var("HOME").unwrap() + "{}/.config/{}/{}", + std::env::var("HOME").unwrap(), + COMMON_DIRECTORY_NAME, + CONFIG_FILE_NAME )) - } else if Path::new("/etc/xdg/websurfx/config.lua").exists() { + } else if Path::new( + format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str(), + ) + .exists() + { Ok("/etc/xdg/websurfx/config.lua".to_string()) - } else if Path::new("./websurfx/config.lua").exists() { + } else if Path::new(format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str()) + .exists() + { Ok("./websurfx/config.lua".to_string()) } else { Err(format!("Config file not found!!").into()) From 29456650f1c86eb6bf690c92b3006ac8d1cfceb7 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 24 May 2023 17:37:41 +0300 Subject: [PATCH 020/392] feat: add ability to put the themes folder on different paths --- src/lib.rs | 16 ++++++++++--- src/server/routes.rs | 4 +++- src/theme_handler/mod.rs | 1 + src/theme_handler/theme_path_handler.rs | 31 +++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 src/theme_handler/mod.rs create mode 100644 src/theme_handler/theme_path_handler.rs diff --git a/src/lib.rs b/src/lib.rs index c234658..0763c96 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ pub mod config_parser; pub mod engines; pub mod search_results_handler; pub mod server; +pub mod theme_handler; use std::net::TcpListener; @@ -15,6 +16,7 @@ use actix_files as fs; use actix_web::{dev::Server, middleware::Logger, web, App, HttpServer}; use config_parser::parser::Config; use handlebars::Handlebars; +use theme_handler::theme_path_handler::handle_different_theme_path; /// Runs the web server on the provided TCP listener and returns a `Server` instance. /// @@ -39,8 +41,10 @@ use handlebars::Handlebars; pub fn run(listener: TcpListener, config: Config) -> std::io::Result { let mut handlebars: Handlebars = Handlebars::new(); + let theme_folder_path: String = handle_different_theme_path()?; + handlebars - .register_templates_directory(".html", "./public/templates") + .register_templates_directory(".html", format!("{}/templates", theme_folder_path)) .unwrap(); let handlebars_ref: web::Data = web::Data::new(handlebars); @@ -51,8 +55,14 @@ pub fn run(listener: TcpListener, config: Config) -> std::io::Result { .app_data(web::Data::new(config.clone())) .wrap(Logger::default()) // added logging middleware for logging. // Serve images and static files (css and js files). - .service(fs::Files::new("/static", "./public/static").show_files_listing()) - .service(fs::Files::new("/images", "./public/images").show_files_listing()) + .service( + fs::Files::new("/static", format!("{}/static", theme_folder_path)) + .show_files_listing(), + ) + .service( + fs::Files::new("/images", format!("{}/images", theme_folder_path)) + .show_files_listing(), + ) .service(routes::robots_data) // robots.txt .service(routes::index) // index page .service(routes::search) // search page diff --git a/src/server/routes.rs b/src/server/routes.rs index 0f84cc9..bb6be27 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -8,6 +8,7 @@ use crate::{ cache::cacher::RedisCache, config_parser::parser::Config, search_results_handler::{aggregation_models::SearchResults, aggregator::aggregate}, + theme_handler::theme_path_handler::handle_different_theme_path, }; use actix_web::{get, web, HttpRequest, HttpResponse}; use handlebars::Handlebars; @@ -146,7 +147,8 @@ pub async fn search( /// Handles the route of robots.txt page of the `websurfx` meta search engine website. #[get("/robots.txt")] pub async fn robots_data(_req: HttpRequest) -> Result> { - let page_content: String = read_to_string("./public/robots.txt")?; + let page_content: String = + read_to_string(format!("{}/robots.txt", handle_different_theme_path()?))?; Ok(HttpResponse::Ok() .content_type("text/plain; charset=ascii") .body(page_content)) diff --git a/src/theme_handler/mod.rs b/src/theme_handler/mod.rs new file mode 100644 index 0000000..3bbca9d --- /dev/null +++ b/src/theme_handler/mod.rs @@ -0,0 +1 @@ +pub mod theme_path_handler; diff --git a/src/theme_handler/theme_path_handler.rs b/src/theme_handler/theme_path_handler.rs new file mode 100644 index 0000000..df3e5cc --- /dev/null +++ b/src/theme_handler/theme_path_handler.rs @@ -0,0 +1,31 @@ +//! This module provides the functionality to handle theme folder present on different paths and +//! provide one appropriate path on which it is present and can be used. + +use std::io::Error; +use std::path::Path; + +// ------- Constants -------- +static THEME_DIRECTORY_NAME: &str = "public"; + +/// A function which returns an appropriate theme directory path checking if the theme +/// directory exists on that path. +/// +/// # Error +/// +/// Returns a `Theme (public) folder not found!!` error if the theme folder is not present under following +/// paths which are: +/// 1. `/opt/websurfx` if it not present here then it fallbacks to the next one (2) +/// 2. Under project folder ( or codebase in other words) if it is not present +/// here then it returns an error as mentioned above. +pub fn handle_different_theme_path() -> Result { + if Path::new(format!("/opt/websurfx/{}/", THEME_DIRECTORY_NAME).as_str()).exists() { + Ok(format!("/opt/websurfx/{}", THEME_DIRECTORY_NAME)) + } else if Path::new(format!("./{}/", THEME_DIRECTORY_NAME).as_str()).exists() { + Ok(format!("./{}", THEME_DIRECTORY_NAME)) + } else { + Err(Error::new( + std::io::ErrorKind::NotFound, + "Themes (public) folder not found!!", + )) + } +} From 42feda0c460fa47d0adf2553197b519aef94c295 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 24 May 2023 17:40:21 +0300 Subject: [PATCH 021/392] chore: Bump version to v0.9.0 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d19acb2..76335f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.8.0" +version = "0.9.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 211f170178b12c6c40a033b7a70a91156132a89f Mon Sep 17 00:00:00 2001 From: neon_arch Date: Wed, 24 May 2023 17:46:35 +0300 Subject: [PATCH 022/392] chore: fix lock error during build --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 02bf14f..ef32619 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1242,9 +1242,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -3319,7 +3319,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.8.0" +version = "0.9.0" dependencies = [ "actix-files", "actix-web", From dc3308cb5f5e28f6dbebe80aafdc2bd2efcf2532 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Thu, 25 May 2023 11:50:37 +0300 Subject: [PATCH 023/392] chore: rename from theme to public --- src/handler/mod.rs | 1 + .../public_path_handler.rs} | 12 ++++++------ src/lib.rs | 12 ++++++------ src/server/routes.rs | 4 ++-- src/theme_handler/mod.rs | 1 - 5 files changed, 15 insertions(+), 15 deletions(-) create mode 100644 src/handler/mod.rs rename src/{theme_handler/theme_path_handler.rs => handler/public_path_handler.rs} (68%) delete mode 100644 src/theme_handler/mod.rs diff --git a/src/handler/mod.rs b/src/handler/mod.rs new file mode 100644 index 0000000..daa5212 --- /dev/null +++ b/src/handler/mod.rs @@ -0,0 +1 @@ +pub mod public_path_handler; diff --git a/src/theme_handler/theme_path_handler.rs b/src/handler/public_path_handler.rs similarity index 68% rename from src/theme_handler/theme_path_handler.rs rename to src/handler/public_path_handler.rs index df3e5cc..b99283e 100644 --- a/src/theme_handler/theme_path_handler.rs +++ b/src/handler/public_path_handler.rs @@ -5,7 +5,7 @@ use std::io::Error; use std::path::Path; // ------- Constants -------- -static THEME_DIRECTORY_NAME: &str = "public"; +static PUBLIC_DIRECTORY_NAME: &str = "public"; /// A function which returns an appropriate theme directory path checking if the theme /// directory exists on that path. @@ -17,11 +17,11 @@ static THEME_DIRECTORY_NAME: &str = "public"; /// 1. `/opt/websurfx` if it not present here then it fallbacks to the next one (2) /// 2. Under project folder ( or codebase in other words) if it is not present /// here then it returns an error as mentioned above. -pub fn handle_different_theme_path() -> Result { - if Path::new(format!("/opt/websurfx/{}/", THEME_DIRECTORY_NAME).as_str()).exists() { - Ok(format!("/opt/websurfx/{}", THEME_DIRECTORY_NAME)) - } else if Path::new(format!("./{}/", THEME_DIRECTORY_NAME).as_str()).exists() { - Ok(format!("./{}", THEME_DIRECTORY_NAME)) +pub fn handle_different_public_path() -> Result { + if Path::new(format!("/opt/websurfx/{}/", PUBLIC_DIRECTORY_NAME).as_str()).exists() { + Ok(format!("/opt/websurfx/{}", PUBLIC_DIRECTORY_NAME)) + } else if Path::new(format!("./{}/", PUBLIC_DIRECTORY_NAME).as_str()).exists() { + Ok(format!("./{}", PUBLIC_DIRECTORY_NAME)) } else { Err(Error::new( std::io::ErrorKind::NotFound, diff --git a/src/lib.rs b/src/lib.rs index 0763c96..6b6d4fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,9 +4,9 @@ pub mod cache; pub mod config_parser; pub mod engines; +pub mod handler; pub mod search_results_handler; pub mod server; -pub mod theme_handler; use std::net::TcpListener; @@ -16,7 +16,7 @@ use actix_files as fs; use actix_web::{dev::Server, middleware::Logger, web, App, HttpServer}; use config_parser::parser::Config; use handlebars::Handlebars; -use theme_handler::theme_path_handler::handle_different_theme_path; +use handler::public_path_handler::handle_different_public_path; /// Runs the web server on the provided TCP listener and returns a `Server` instance. /// @@ -41,10 +41,10 @@ use theme_handler::theme_path_handler::handle_different_theme_path; pub fn run(listener: TcpListener, config: Config) -> std::io::Result { let mut handlebars: Handlebars = Handlebars::new(); - let theme_folder_path: String = handle_different_theme_path()?; + let public_folder_path: String = handle_different_public_path()?; handlebars - .register_templates_directory(".html", format!("{}/templates", theme_folder_path)) + .register_templates_directory(".html", format!("{}/templates", public_folder_path)) .unwrap(); let handlebars_ref: web::Data = web::Data::new(handlebars); @@ -56,11 +56,11 @@ pub fn run(listener: TcpListener, config: Config) -> std::io::Result { .wrap(Logger::default()) // added logging middleware for logging. // Serve images and static files (css and js files). .service( - fs::Files::new("/static", format!("{}/static", theme_folder_path)) + fs::Files::new("/static", format!("{}/static", public_folder_path)) .show_files_listing(), ) .service( - fs::Files::new("/images", format!("{}/images", theme_folder_path)) + fs::Files::new("/images", format!("{}/images", public_folder_path)) .show_files_listing(), ) .service(routes::robots_data) // robots.txt diff --git a/src/server/routes.rs b/src/server/routes.rs index bb6be27..ead1612 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -7,8 +7,8 @@ use std::fs::read_to_string; use crate::{ cache::cacher::RedisCache, config_parser::parser::Config, + handler::public_path_handler::handle_different_public_path, search_results_handler::{aggregation_models::SearchResults, aggregator::aggregate}, - theme_handler::theme_path_handler::handle_different_theme_path, }; use actix_web::{get, web, HttpRequest, HttpResponse}; use handlebars::Handlebars; @@ -148,7 +148,7 @@ pub async fn search( #[get("/robots.txt")] pub async fn robots_data(_req: HttpRequest) -> Result> { let page_content: String = - read_to_string(format!("{}/robots.txt", handle_different_theme_path()?))?; + read_to_string(format!("{}/robots.txt", handle_different_public_path()?))?; Ok(HttpResponse::Ok() .content_type("text/plain; charset=ascii") .body(page_content)) diff --git a/src/theme_handler/mod.rs b/src/theme_handler/mod.rs deleted file mode 100644 index 3bbca9d..0000000 --- a/src/theme_handler/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod theme_path_handler; From 6556e0fdad13db4fd30950a5fca9da2a9ba43839 Mon Sep 17 00:00:00 2001 From: LX5321 Date: Fri, 26 May 2023 13:08:17 +0530 Subject: [PATCH 024/392] UPDATED: Removed Deprecated API --- public/static/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/static/index.js b/public/static/index.js index 1261e15..ad52ab9 100644 --- a/public/static/index.js +++ b/public/static/index.js @@ -4,7 +4,7 @@ function search_web() { } search_box.addEventListener('keyup', (e) => { - if (e.keyCode === 13) { + if (e.key === 'Enter') { search_web() } }) From 5003c8e61f9fbf1ab2c7551d7794731a6d3e2bd7 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Fri, 26 May 2023 19:27:15 +0300 Subject: [PATCH 025/392] chore: reorder config file options under appropriate headings and make headings standout --- websurfx/config.lua | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/websurfx/config.lua b/websurfx/config.lua index c30f376..595627e 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -1,8 +1,11 @@ --- Server +-- ### Server ### port = "8080" -- port on which server should be launched binding_ip_addr = "127.0.0.1" --ip address on the which server should be launched. +production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) +-- if production_use is set to true +-- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. --- Website +-- ### Website ### -- The different colorschemes provided are: -- {{ -- catppuccin-mocha @@ -17,9 +20,5 @@ binding_ip_addr = "127.0.0.1" --ip address on the which server should be launche colorscheme = "catppuccin-mocha" -- the colorscheme name which should be used for the website theme theme = "simple" -- the theme name which should be used for the website --- Caching +-- ### Caching ### redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. - -production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) --- if production_use is set to true - -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. \ No newline at end of file From f7c1df268fd74196da561ed9613a344b475adda7 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sat, 27 May 2023 17:01:47 +0300 Subject: [PATCH 026/392] chore: fix the docs link --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index be44535..324dfbf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ ## Documentation/Wiki -Found a typo, or something that isn't as clear as it could be? Maybe I've missed something off altogether, or you hit a roadblock that took you a while to figure out. Edit the [wiki](https://github.com/neon-mmd/websurfx/wiki) to add to or improve the documentation. This will help future users get Websurfx up and running more easily. +Found a typo, or something that isn't as clear as it could be? Maybe I've missed something off altogether, or you hit a roadblock that took you a while to figure out. Edit the [docs](./docs/) to add to or improve the documentation. This will help future users get Websurfx up and running more easily. ## Readme From 87ac0b7cfe90b941a9b1758766c8498358459177 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sat, 27 May 2023 17:06:45 +0300 Subject: [PATCH 027/392] docs: fix the installation instructions --- docs/installation.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 4719ddc..d8f301b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -34,6 +34,7 @@ To get started with Websurfx, clone the repository, edit the config file which i ```shell git clone https://github.com/neon-mmd/websurfx.git cd websurfx +git checkout stable cargo build -r redis-server --port 8082 & ./target/release/websurfx @@ -50,7 +51,6 @@ If you want to use the rolling/edge branch, run the following commands instead: ```shell git clone https://github.com/neon-mmd/websurfx.git cd websurfx -git checkout rolling cargo build -r redis-server --port 8082 & ./target/release/websurfx @@ -64,7 +64,7 @@ If you want to change the port or the ip or any other configuration setting chec Before you start, you will need [Docker](https://docs.docker.com/get-docker/) installed on your system first. -## Stable +## Unstable/Edge/Rolling First clone the the repository by running the following command: @@ -111,14 +111,14 @@ docker compose up -d --build This will take around 5-10 mins for first deployment, afterwards the docker build stages will be cached so it will be faster to be build from next time onwards. After the above step finishes launch your preferred browser and then navigate to `http://:`. -## Unstable/Edge/Rolling +## Stable -For the unstable/rolling/edge version, follow the same steps as above (as mentioned for the stable version) with an addition of one command which has to be performed after cloning and changing directory into the repository which makes the cloning step as follows: +For the stable version, follow the same steps as above (as mentioned for the unstable/rolling/edge version) with an addition of one command which has to be performed after cloning and changing directory into the repository which makes the cloning step as follows: ```bash git clone https://github.com/neon-mmd/websurfx.git cd websurfx -git checkout rolling +git checkout stable ``` [⬅️ Go back to Home](./README.md) From 1d638ffeea75d7d1a86a9c07c04d1df5006d48d6 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sat, 27 May 2023 17:07:26 +0300 Subject: [PATCH 028/392] chore: fix the installation instructions under installation section --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 95d4d21..36b358b 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ To get started with Websurfx, clone the repository, edit the config file, which ``` shell git clone https://github.com/neon-mmd/websurfx.git cd websurfx +git checkout stable cargo build -r redis-server --port 8082 & ./target/release/websurfx From 294dfe74a9d21cfeb9a1cf6a8c86b9f0a45bf150 Mon Sep 17 00:00:00 2001 From: Sam sunder Date: Sat, 27 May 2023 15:01:52 +0000 Subject: [PATCH 029/392] Centered About page content --- public/static/themes/simple.css | 1 + 1 file changed, 1 insertion(+) diff --git a/public/static/themes/simple.css b/public/static/themes/simple.css index b9390e7..4ff3355 100644 --- a/public/static/themes/simple.css +++ b/public/static/themes/simple.css @@ -296,4 +296,5 @@ footer { .about-container h3{ font-size: 1.5rem; + width: 80%; } From 6006a983b331af9634c69c7d73407ae54dbb8454 Mon Sep 17 00:00:00 2001 From: Sam sunder Date: Sat, 27 May 2023 15:38:47 +0000 Subject: [PATCH 030/392] Centered about div 2nd commit --- public/static/themes/simple.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/public/static/themes/simple.css b/public/static/themes/simple.css index 4ff3355..17962d0 100644 --- a/public/static/themes/simple.css +++ b/public/static/themes/simple.css @@ -296,5 +296,8 @@ footer { .about-container h3{ font-size: 1.5rem; + } + + .about-container { width: 80%; } From 8e7dc68d2da3d7d2f79ed652b67d44e7ad018564 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sat, 27 May 2023 19:50:20 +0300 Subject: [PATCH 031/392] feat: add an option to enable or disable logs --- src/bin/websurfx.rs | 6 ++++-- src/config_parser/parser.rs | 2 ++ websurfx/config.lua | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index fa21486..ca05713 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -5,7 +5,6 @@ use std::net::TcpListener; -use env_logger::Env; use websurfx::{config_parser::parser::Config, run}; /// The function that launches the main server and registers all the routes of the website. @@ -20,7 +19,10 @@ async fn main() -> std::io::Result<()> { let config = Config::parse().unwrap(); // Initializing logging middleware with level set to default or info. - env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); + if config.logging { + use env_logger::Env; + env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); + } log::info!("started server on port {}", config.port); diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index 55d4bec..dd92f1b 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -25,6 +25,7 @@ pub struct Config { pub style: Style, pub redis_connection_url: String, pub aggregator: AggreatorConfig, + pub logging: bool, } /// Configuration options for the aggregator. @@ -71,6 +72,7 @@ impl Config { ), redis_connection_url: globals.get::<_, String>("redis_connection_url")?, aggregator: aggregator_config, + logging: globals.get::<_, bool>("logging")?, }) }) } diff --git a/websurfx/config.lua b/websurfx/config.lua index 595627e..29c4fff 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -1,3 +1,6 @@ +-- ### General ### +logging = true -- an option to enable or disable logs. + -- ### Server ### port = "8080" -- port on which server should be launched binding_ip_addr = "127.0.0.1" --ip address on the which server should be launched. From 473ada2871fb41a6c5a36de0dcfcd5863b548ed0 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 28 May 2023 11:23:11 +0300 Subject: [PATCH 032/392] docs: update docs to explain about the new option logging --- docs/configuration.md | 5 +++++ docs/installation.md | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bb10ba6..f700389 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -13,10 +13,15 @@ If you have installed `websurfx` using the package manager of your Linux distro Some of the configuration options provided in the file are stated below. These are subdivided into three categories: +- General - Server - Website - Cache +# General + +- **logging:** An option to enable or disable logs. + ## Server - **port:** Port number on which server should be launched. diff --git a/docs/installation.md b/docs/installation.md index d8f301b..1a04254 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -76,11 +76,17 @@ cd websurfx After that edit the config.lua file located under `websurfx` directory. In the config file you will specifically need to change to values which is `binding_ip_addr` and `redis_connection_url` which should make the config look something like this: ```lua --- Server +-- ### General ### +logging = true -- an option to enable or disable logs. + +-- ### Server ### port = "8080" -- port on which server should be launched binding_ip_addr = "0.0.0.0" --ip address on the which server should be launched. +production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) +-- if production_use is set to true +-- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. --- Website +-- ### Website ### -- The different colorschemes provided are: -- {{ -- catppuccin-mocha @@ -95,12 +101,8 @@ binding_ip_addr = "0.0.0.0" --ip address on the which server should be launched. colorscheme = "catppuccin-mocha" -- the colorscheme name which should be used for the website theme theme = "simple" -- the theme name which should be used for the website --- Caching -redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. - -production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) --- if production_use is set to true - -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. +-- ### Caching ### +redis_connection_url = "redis://redis:6379" -- redis connection url address on which the client should connect on. ``` After this run the following command to deploy the app: From 68da0d854aed612041dc2f183d19b7e2d8b9a1ed Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 28 May 2023 11:24:28 +0300 Subject: [PATCH 033/392] docs: improve the wording of the configuration page --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index f700389..a5391a3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,7 +11,7 @@ If you have built `websurfx` from source then the configuration file will be loc If you have installed `websurfx` using the package manager of your Linux distro then the default configuration file will be located at `/etc/xdg/websurfx/`. You can copy the default config to `~/.config/websurfx/` and make the changes there and rerun the websurfx server. -Some of the configuration options provided in the file are stated below. These are subdivided into three categories: +Some of the configuration options provided in the file are stated below. These are subdivided into the following categories: - General - Server From 7aa1394e7c6ebec20cc9f33660d6009311d719fc Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 28 May 2023 11:30:08 +0300 Subject: [PATCH 034/392] chore: bump version to v0.10.0 --- Cargo.lock | 157 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 78 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef32619..e35e94c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ dependencies = [ "futures-sink", "memchr", "pin-project-lite", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-util", "tracing", ] @@ -53,7 +53,7 @@ dependencies = [ "actix-service", "actix-utils", "ahash 0.8.3", - "base64 0.21.1", + "base64 0.21.2", "bitflags", "brotli", "bytes 1.4.0", @@ -75,7 +75,7 @@ dependencies = [ "rand 0.8.5", "sha1", "smallvec 1.10.0", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-util", "tracing", "zstd", @@ -87,7 +87,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "syn 1.0.109", ] @@ -111,7 +111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" dependencies = [ "futures-core", - "tokio 1.28.1", + "tokio 1.28.2", ] [[package]] @@ -128,7 +128,7 @@ dependencies = [ "mio 0.8.6", "num_cpus", "socket2", - "tokio 1.28.1", + "tokio 1.28.2", "tracing", ] @@ -201,8 +201,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" dependencies = [ "actix-router", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -315,9 +315,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1e31e207a6b8fb791a38ea3105e6cb541f55e4d029902d3039a4ad07cc4105" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "bit-set" @@ -605,8 +605,8 @@ dependencies = [ "itoa 1.0.6", "matches", "phf 0.10.1", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "smallvec 1.10.0", "syn 1.0.109", ] @@ -617,7 +617,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "syn 1.0.109", ] @@ -628,8 +628,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -730,8 +730,8 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", "synstructure", ] @@ -971,7 +971,7 @@ dependencies = [ "http 0.2.9", "indexmap", "slab", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-util", "tracing", ] @@ -1035,8 +1035,8 @@ dependencies = [ "log", "mac", "markup5ever 0.11.0", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -1157,7 +1157,7 @@ dependencies = [ "itoa 1.0.6", "pin-project-lite", "socket2", - "tokio 1.28.1", + "tokio 1.28.2", "tower-service", "tracing", "want 0.3.0", @@ -1185,7 +1185,7 @@ dependencies = [ "bytes 1.4.0", "hyper 0.14.26", "native-tls", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-native-tls", ] @@ -1381,12 +1381,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" [[package]] name = "mac" @@ -1631,9 +1628,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -1749,9 +1746,9 @@ checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -1863,8 +1860,8 @@ dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", "proc-macro-hack", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -1942,9 +1939,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" dependencies = [ "unicode-ident", ] @@ -1970,11 +1967,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ - "proc-macro2 1.0.58", + "proc-macro2 1.0.59", ] [[package]] @@ -2213,9 +2210,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.2" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1a59b5d8e97dee33696bf13c5ba8ab85341c002922fba050069326b9c498974" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" dependencies = [ "aho-corasick", "memchr", @@ -2268,7 +2265,7 @@ version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.1", + "base64 0.21.2", "bytes 1.4.0", "encoding_rs", "futures-core", @@ -2289,7 +2286,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.1", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-native-tls", "tower-service", "url 2.3.1", @@ -2495,9 +2492,9 @@ version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2680,8 +2677,8 @@ checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" dependencies = [ "phf_generator 0.7.24", "phf_shared 0.7.24", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "string_cache_shared", ] @@ -2693,8 +2690,8 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", ] [[package]] @@ -2720,19 +2717,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.16" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "unicode-ident", ] @@ -2742,8 +2739,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -2796,9 +2793,9 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2875,9 +2872,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.1" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg 1.1.0", "bytes 1.4.0", @@ -2940,9 +2937,9 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2952,7 +2949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.28.1", + "tokio 1.28.2", ] [[package]] @@ -3037,7 +3034,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.28.1", + "tokio 1.28.2", "tracing", ] @@ -3112,9 +3109,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-normalization" @@ -3260,9 +3257,9 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", "wasm-bindgen-shared", ] @@ -3284,7 +3281,7 @@ version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "wasm-bindgen-macro-support", ] @@ -3294,9 +3291,9 @@ version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3319,7 +3316,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.9.0" +version = "0.10.0" dependencies = [ "actix-files", "actix-web", @@ -3336,7 +3333,7 @@ dependencies = [ "scraper", "serde", "serde_json", - "tokio 1.28.1", + "tokio 1.28.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 76335f8..8d8d452 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.9.0" +version = "0.10.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 6ef6b323256220a5a84b6bc3ad7d089ea4654af4 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 29 May 2023 19:45:00 +0300 Subject: [PATCH 035/392] ci: add CI to automate the generation of contributors list --- .github/workflows/contributors.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/contributors.yml diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml new file mode 100644 index 0000000..bed59af --- /dev/null +++ b/.github/workflows/contributors.yml @@ -0,0 +1,16 @@ +name: Contributors +on: + schedule: + - cron: '0 1 * * *' # At 01:00 on every day. + push: + branches: + - rolling +jobs: + contributors: + runs-on: ubuntu-latest + steps: + - uses: wow-actions/contributors-list@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + round: true + svgPath: ../../images/contributors_list.svg From d4df90160d276b50a7b9ebf2728407574ae5d639 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 29 May 2023 21:09:07 +0300 Subject: [PATCH 036/392] feat: add an option to enable/disable debug mode --- src/bin/websurfx.rs | 2 +- src/config_parser/parser.rs | 2 ++ websurfx/config.lua | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index ca05713..91708aa 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -19,7 +19,7 @@ async fn main() -> std::io::Result<()> { let config = Config::parse().unwrap(); // Initializing logging middleware with level set to default or info. - if config.logging { + if config.logging || config.debug{ use env_logger::Env; env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); } diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index dd92f1b..ac200cd 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -26,6 +26,7 @@ pub struct Config { pub redis_connection_url: String, pub aggregator: AggreatorConfig, pub logging: bool, + pub debug: bool, } /// Configuration options for the aggregator. @@ -73,6 +74,7 @@ impl Config { redis_connection_url: globals.get::<_, String>("redis_connection_url")?, aggregator: aggregator_config, logging: globals.get::<_, bool>("logging")?, + debug: globals.get::<_, bool>("debug")?, }) }) } diff --git a/websurfx/config.lua b/websurfx/config.lua index 29c4fff..3daaa91 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -1,5 +1,6 @@ -- ### General ### logging = true -- an option to enable or disable logs. +debug = false -- an option to enable or disable debug mode. -- ### Server ### port = "8080" -- port on which server should be launched From 60317a3b75cbd256d3dd2b1723a31aa82bd051ef Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 29 May 2023 21:13:07 +0300 Subject: [PATCH 037/392] chore: make format ci happy --- src/bin/websurfx.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index 91708aa..8661725 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -19,7 +19,7 @@ async fn main() -> std::io::Result<()> { let config = Config::parse().unwrap(); // Initializing logging middleware with level set to default or info. - if config.logging || config.debug{ + if config.logging || config.debug { use env_logger::Env; env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); } From eb44ff7cfa3a2736720052c64cd2f8fd2baca40c Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 29 May 2023 21:17:00 +0300 Subject: [PATCH 038/392] chore: Bump version to v0.11.0 --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e35e94c..9af7c18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,7 +125,7 @@ dependencies = [ "actix-utils", "futures-core", "futures-util", - "mio 0.8.6", + "mio 0.8.7", "num_cpus", "socket2", "tokio 1.28.2", @@ -1510,14 +1510,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "eebffdb73fe72e917997fad08bdbf31ac50b0fa91cec93e69a0662e4264d454c" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -1603,9 +1603,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" [[package]] name = "openssl" @@ -2879,7 +2879,7 @@ dependencies = [ "autocfg 1.1.0", "bytes 1.4.0", "libc", - "mio 0.8.6", + "mio 0.8.7", "num_cpus", "parking_lot 0.12.1", "pin-project-lite", @@ -3316,7 +3316,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.10.0" +version = "0.11.0" dependencies = [ "actix-files", "actix-web", diff --git a/Cargo.toml b/Cargo.toml index 8d8d452..17aa0a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.10.0" +version = "0.11.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 13632f1f99bbabdbff503729a3238798d000c2bd Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 29 May 2023 21:28:09 +0300 Subject: [PATCH 039/392] feat: remove random delays when debug is set to true --- src/search_results_handler/aggregator.rs | 3 ++- src/server/routes.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/search_results_handler/aggregator.rs b/src/search_results_handler/aggregator.rs index 8b86972..8b6bae3 100644 --- a/src/search_results_handler/aggregator.rs +++ b/src/search_results_handler/aggregator.rs @@ -40,12 +40,13 @@ pub async fn aggregate( query: &str, page: u32, random_delay: bool, + debug: bool, ) -> Result> { let user_agent: String = random_user_agent(); let mut result_map: HashMap = HashMap::new(); // Add a random delay before making the request. - if random_delay { + if random_delay || !debug { let mut rng = rand::thread_rng(); let delay_secs = rng.gen_range(1..10); std::thread::sleep(Duration::from_secs(delay_secs)); diff --git a/src/server/routes.rs b/src/server/routes.rs index ead1612..9234d8d 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -128,7 +128,7 @@ pub async fn search( } Err(_) => { let mut results_json: crate::search_results_handler::aggregation_models::SearchResults = - aggregate(query, page, config.aggregator.random_delay).await?; + aggregate(query, page, config.aggregator.random_delay, config.debug).await?; results_json.add_style(config.style.clone()); redis_cache .cache_results(serde_json::to_string(&results_json)?, &page_url)?; From af8b3ce71b0b5d3388fe0ca009f22271aff19877 Mon Sep 17 00:00:00 2001 From: XFFXFF <1247714429@qq.com> Date: Tue, 30 May 2023 09:23:42 +0000 Subject: [PATCH 040/392] fix the contributor list action --- .github/workflows/contributors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index bed59af..3d8580e 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -13,4 +13,4 @@ jobs: with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} round: true - svgPath: ../../images/contributors_list.svg + svgPath: images/contributors_list.svg From aeb2510de8c07f46ef09c09eb9f3ec8dfebc6806 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 30 May 2023 13:00:08 +0300 Subject: [PATCH 041/392] ci: fix contributors list autogeneration by appropriate permissions --- .github/workflows/contributors.yml | 45 +++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index bed59af..7d990f5 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -1,16 +1,47 @@ -name: Contributors +name: Contributors List + on: + workflow_dispatch: + schedule: - - cron: '0 1 * * *' # At 01:00 on every day. - push: - branches: - - rolling + - cron: "0 1 * * *" + jobs: contributors: + permissions: + contents: write + pull-requests: write + runs-on: ubuntu-latest + steps: - - uses: wow-actions/contributors-list@v1 + - name: Checkout code + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + with: + fetch-depth: 0 + ref: ${{ github.event.repository.default_branch }} + + - name: Update contributors list + uses: wow-actions/contributors-list@b9e91f91a51a55460fdcae64daad0cb8122cdd53 # v1.1.0 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + svgPath: images/contributors_list.svg round: true - svgPath: ../../images/contributors_list.svg + includeBots: false + noCommit: true + + - name: Commit & PR + uses: peter-evans/create-pull-request@38e0b6e68b4c852a5500a94740f0e535e0d7ba54 # v4.2.4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + add-paths: .github/assets/CONTRIBUTORS.svg + commit-message: 'chore: update contributors-list' + committer: GitHub + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: false + branch: workflow/update-contributors-list + base: main + delete-branch: true + title: 'chore: update contributors-list' + body: | + Automated update to `images/contributors_list.svg` From af145e9d85249df3f89f76dd48f4b6a6255f8c24 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 30 May 2023 14:55:11 +0300 Subject: [PATCH 042/392] docs: update docs to explain about the new option debug --- docs/configuration.md | 1 + docs/installation.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index a5391a3..0d54fd9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,6 +21,7 @@ Some of the configuration options provided in the file are stated below. These a # General - **logging:** An option to enable or disable logs. +- **debug:** An option to enable or disable debug mode. ## Server diff --git a/docs/installation.md b/docs/installation.md index 1a04254..fbb0d16 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -78,6 +78,7 @@ After that edit the config.lua file located under `websurfx` directory. In the c ```lua -- ### General ### logging = true -- an option to enable or disable logs. +debug = false -- an option to enable or disable debug mode. -- ### Server ### port = "8080" -- port on which server should be launched From 8c4d359e26aac0c8690c993362423794e20ae52c Mon Sep 17 00:00:00 2001 From: MD AL AMIN TALUKDAR <129589283+alamin655@users.noreply.github.com> Date: Wed, 31 May 2023 16:48:50 +0530 Subject: [PATCH 043/392] Create tokyo-night.css --- public/static/colorschemes/tokyo-night.css | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 public/static/colorschemes/tokyo-night.css diff --git a/public/static/colorschemes/tokyo-night.css b/public/static/colorschemes/tokyo-night.css new file mode 100644 index 0000000..b7a30cf --- /dev/null +++ b/public/static/colorschemes/tokyo-night.css @@ -0,0 +1,11 @@ +:root { + --bg: #1a1b26; + --fg: #c0caf5; + --1: #32364a; + --2: #a9b1d6; + --3: #5a5bb8; + --4: #6b7089; + --5: #e2afff; + --6: #a9a1e1; + --7: #988bc7; +} From bc43e15467af3917067085dd451a3fd15c4a3379 Mon Sep 17 00:00:00 2001 From: MD AL AMIN TALUKDAR <129589283+alamin655@users.noreply.github.com> Date: Wed, 31 May 2023 16:50:13 +0530 Subject: [PATCH 044/392] Create one-dark.css --- public/static/colorschemes/one-dark.css | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 public/static/colorschemes/one-dark.css diff --git a/public/static/colorschemes/one-dark.css b/public/static/colorschemes/one-dark.css new file mode 100644 index 0000000..0afb05e --- /dev/null +++ b/public/static/colorschemes/one-dark.css @@ -0,0 +1,11 @@ +:root { + --bg: #282c34; + --fg: #abb2bf; + --1: #3b4048; + --2: #a3be8c; + --3: #b48ead; + --4: #c8ccd4; + --5: #e06c75; + --6: #61afef; + --7: #be5046; +} From 59013c83e4fe6595651da5c77a4a6aeeeaab4003 Mon Sep 17 00:00:00 2001 From: MD AL AMIN TALUKDAR <129589283+alamin655@users.noreply.github.com> Date: Wed, 31 May 2023 16:51:00 +0530 Subject: [PATCH 045/392] Create dark-chocolate.css --- public/static/colorschemes/dark-chocolate.css | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 public/static/colorschemes/dark-chocolate.css diff --git a/public/static/colorschemes/dark-chocolate.css b/public/static/colorschemes/dark-chocolate.css new file mode 100644 index 0000000..f1d6848 --- /dev/null +++ b/public/static/colorschemes/dark-chocolate.css @@ -0,0 +1,11 @@ +:root { + --bg: #000000; + --fg: #ffffff; + --1: #121212; + --2: #808080; + --3: #999999; + --4: #666666; + --5: #bfbfbf; + --6: #e0e0e0; + --7: #555555; + } From a98f3fa2f5e5239e119edd657290f9cc7d3d9d35 Mon Sep 17 00:00:00 2001 From: Sam sunder Date: Wed, 31 May 2023 18:35:02 +0530 Subject: [PATCH 046/392] Added settings in settings page and added search link to navbar --- .../static/colorschemes/catppuccin-mocha.css | 2 + public/static/settings.js | 173 +++++++++++++ public/static/themes/simple.css | 233 +++++++++++++++--- public/templates/navbar.html | 1 + public/templates/settings.html | 85 ++++++- 5 files changed, 462 insertions(+), 32 deletions(-) create mode 100644 public/static/settings.js diff --git a/public/static/colorschemes/catppuccin-mocha.css b/public/static/colorschemes/catppuccin-mocha.css index d2c1075..6432155 100644 --- a/public/static/colorschemes/catppuccin-mocha.css +++ b/public/static/colorschemes/catppuccin-mocha.css @@ -2,8 +2,10 @@ --bg: #1e1e2e; --fg: #cdd6f4; --1: #45475a; + --1_1: #4f5169; --2: #f38ba8; --3: #a6e3a1; + --3_1: #7ce073; --4: #f9e2af; --5: #89b4fa; --6: #f5c2e7; diff --git a/public/static/settings.js b/public/static/settings.js new file mode 100644 index 0000000..0af04bf --- /dev/null +++ b/public/static/settings.js @@ -0,0 +1,173 @@ +function toggleSelectOptions(elem, state) { + elem.classList.remove("invalid"); + try { elem.parentElement.querySelector('.errTxt').remove();} catch (error) {} + let options = elem.querySelector('.options'); + const pos = elem.getBoundingClientRect(); + const windowWidth = document.getElementsByTagName("body")[0].clientHeight; + if(pos.y + 250 > windowWidth) { + options.style.bottom = '40px'; + } else { options.style.bottom = null } + options.style.display = state != 'close' ? getComputedStyle(options).display == 'none' ? 'block': 'none' : 'none'; +} + +let selectElements = document.querySelectorAll('.custom-select'); +Array.from(selectElements).forEach(element => { + element.childNodes[0].nodeValue = (element.hasAttribute('data-multiple') ? element.getAttribute('data-placeholder') : element.getAttribute('data-default')); + element.addEventListener('click', (e) => {if (e.target === element) toggleSelectOptions(element)}); + element.addEventListener('focusout', (e) => {if (e.target === element) toggleSelectOptions(element, 'close')}); +}); + +function removeSelectOption(elem, optionId) { + let option = document.querySelector('#'+optionId); + let selectDiv = option.closest('.custom-select'); + let input = document.querySelector(`[name="${selectDiv.getAttribute('data-input')}"]`); + elem.parentElement.remove(); + option.removeAttribute('selected'); + option.querySelector('svg').remove(); + input.value = input.value.replace(option.getAttribute('data-value') ? option.getAttribute('data-value') + "," : '', ''); +} + +function multiSelectClickHandler(elem) { + let selectDiv = elem.closest('.custom-select'); + let input = document.querySelector(`[name="${selectDiv.getAttribute('data-input')}"]`); + if (!elem.hasAttribute('selected')) { + document.querySelector('#'+elem.closest(".custom-select").getAttribute("data-showDivId")).innerHTML += + `${elem.innerText}   + + + + + `; + elem.setAttribute('selected', ''); + elem.innerHTML = `${elem.innerText} + + ` + + // Code where value in inserted into input + input.value += elem.getAttribute('data-value') ? elem.getAttribute('data-value') + "," : ''; + } else { + // similar to removeSelectOption method + document.querySelector('#'+elem.getAttribute('id')+'-selected').remove(); + elem.removeAttribute('selected'); + elem.querySelector('svg').remove(); + + input.value = input.value.replace(elem.getAttribute('data-value') ? elem.getAttribute('data-value') + "," : '', ''); + } + +} + +function singleSelectClickHandler(elem) { + let selectDiv = elem.closest('.custom-select'); + let selectedOption = selectDiv.querySelector('span[selected]'); + let input = document.querySelector(`[name="${selectDiv.getAttribute('data-input')}"]`); + if (!elem.hasAttribute('selected')) { + if (selectedOption != null) { + selectedOption.removeAttribute('selected'); + selectedOption.querySelector('svg').remove(); + } + elem.setAttribute('selected', ''); + elem.innerHTML = `${elem.innerText} + + ` + // Code where value is inserted to input + input.value = elem.getAttribute('data-value') ? elem.getAttribute('data-value') : ''; + selectDiv.childNodes[0].nodeValue = elem.innerText; + } else { + elem.removeAttribute('selected'); + elem.querySelector('svg').remove(); + selectDiv.childNodes[0].nodeValue = selectDiv.getAttribute('data-default'); + + input.value = ""; + } + selectDiv.blur(); +} + +let multiSelectOptions = document.querySelectorAll('.custom-select[data-multiple="1"]>.options span'); +for (let i = 0; i < multiSelectOptions.length; i++) { + multiSelectOptions[i].addEventListener('click', () => {multiSelectClickHandler(multiSelectOptions[i])}); + multiSelectOptions[i].setAttribute('id', 'option-'+i.toString()); +} + +let singleSelectOptions = document.querySelectorAll('.custom-select:not([data-multiple="1"])>.options span'); +for (let i = 0; i < singleSelectOptions.length; i++) { + singleSelectOptions[i].addEventListener('click', () => {singleSelectClickHandler(singleSelectOptions[i])}); + singleSelectOptions[i].setAttribute('id', 'option-'+i.toString()); +} + +function selectAllHandler(elem) { + let options = elem.parentElement.querySelectorAll('.custom-select[data-multiple="1"]>.options span'); + Array.from(options).forEach((option) => { + try { + let selectedShownElem = document.getElementById(option.getAttribute('id')+'-selected'); + removeSelectOption(selectedShownElem.querySelector('span'), option.getAttribute('id')) + } catch (err) {} + if (elem.innerText == 'select all') { option.click() }; + }); + elem.innerText = elem.innerText == 'select all' ? 'deselect all' : 'select all' +} + + +// On settings form submit +function submitSettings() { + let form = document.settings; + document.querySelectorAll('.errTxt').forEach(e => e.remove()); + for(let i = 0; i < form.elements.length; i++) { + let input = form.elements[i]; + if (input.value == "" && input.hasAttribute('required')) { + let elem = input.parentElement.querySelector('[takeInput]') + let errTxt = document.createElement('p') + errTxt.classList.add('errTxt') + errTxt.innerText = 'This setting can\'t be empty!!' + elem.classList.add('invalid'); + elem.parentElement.insertBefore(errTxt, elem); + } + } + return false; +} + +document.querySelectorAll('.settings-sidebar .set-name').forEach(filter => { + let target = filter.getAttribute('data-detailId'); + filter.addEventListener('click', () => { + try {document.querySelector('.set-name.active').classList.remove('active');} catch(e){} + filter.classList.add('active'); + if (target == 'all') { + document.querySelectorAll('.set-item').forEach((elem) => { + elem.style.display = 'block'; + }) + return; + } + document.querySelectorAll('.set-item[data-id="'+target+'"]').forEach((elem) => { + elem.style.display = 'block' + }) + document.querySelectorAll('.set-item:not([data-id="'+target+'"])').forEach((elem) => { + elem.style.display = 'none' + }) + }) +}) + +function fadeOut(element) { + var op = 1; // initial opacity + var timer = setInterval(function () { + if (op <= 0.1){ + clearInterval(timer); + element.style.display = 'none'; + element.classList.add('fade'); + } + element.style.opacity = op; + element.style.filter = 'alpha(opacity=' + op * 100 + ")"; + op -= op * 0.1; + }, 50); +} + +function fadeIn(element) { + var op = 0.1; // initial opacity + element.style.display = 'block'; + var timer = setInterval(function () { + if (op >= 1){ + clearInterval(timer); + } + element.style.opacity = op; + element.style.filter = 'alpha(opacity=' + op * 100 + ")"; + op += op * 0.1; + }, 10); +} diff --git a/public/static/themes/simple.css b/public/static/themes/simple.css index 17962d0..61776ae 100644 --- a/public/static/themes/simple.css +++ b/public/static/themes/simple.css @@ -263,41 +263,212 @@ footer { /* Styles for the about page */ - .about-container article{ - font-size: 1.5rem; - color:var(--fg); - padding-bottom: 10px; - } +.about-container article{ + font-size: 1.5rem; + color:var(--fg); + padding-bottom: 10px; +} - .about-container article h1{ - color: var(--2); - font-size: 2.8rem; - } +.about-container article h1{ + color: var(--2); + font-size: 2.8rem; +} - .about-container article div{ - padding-bottom: 15px; - } +.about-container article div{ + padding-bottom: 15px; +} - .about-container a{ - color:var(--3); - } +.about-container a{ + color:var(--3); +} - .about-container article h2{ - color: var(--3); - font-size: 1.8rem; - padding-bottom: 10px; - } +.about-container article h2{ + color: var(--3); + font-size: 1.8rem; + padding-bottom: 10px; +} - .about-container p{ - color:var(--fg); - font-size: 1.6rem; - padding-bottom: 10px; - } +.about-container p{ + color:var(--fg); + font-size: 1.6rem; + padding-bottom: 10px; +} - .about-container h3{ - font-size: 1.5rem; - } +.about-container h3{ + font-size: 1.5rem; +} - .about-container { - width: 80%; - } +.about-container { + width: 80%; +} + +.settings { + margin: 2rem; + width: 80%; + height: 100%; + color: var(--fg); +} + +.settings h1 { + color: var(--2); + font-size: 2.5rem; +} + + +.settings hr { + border-color: var(--3); + margin: .3rem 0 1rem 0; +} + +.settings-view { + display: flex; + flex: 1 0 auto; +} + +.settings-sidebar { + width: 25%; + height: 100%; +} + +.settings-sidebar .set-name { + cursor: pointer; + font-size: 2rem; + display: block; + margin-right: .5rem; + margin-left: -.7rem; + padding: .7rem; + border-radius: 5px; + font-weight: bold; +} + +.settings-sidebar .set-name:hover, .settings-sidebar .set-name.active { + background-color: var(--1_1); +} + +.settings-detail { + border-left: 1.5px solid var(--3); + padding-left: 3rem; + margin-top: .7rem; +} + +.settings-detail .set-item { + margin: 2rem 0; + margin-top: 0; +} + +.settings-detail .set-name { + font-size: 2rem; + font-weight: bold; + color: var(--4) +} + +.settings-detail .set-desc { + font-size: 1.5rem; + margin-bottom: .5rem; +} + +.custom-select, .options { + font-size: 1.5rem; + background-color: var(--bg); + width: 250px; + padding: 1rem 1.7rem; + border-radius: 7px; +} + +.custom-select { + position: relative; + vertical-align: middle; + margin: .7rem 0; +} + +.custom-select.invalid { + border: 1px solid red; +} + +.custom-select svg { + float: right; +} + +.options { + display: none; + position: absolute; + left: 0; + margin-top: 1.3rem; + width: 100%; + padding: .7rem 1rem; + cursor: pointer; + z-index: 3; + max-height: 15rem; + overflow-y: auto; +} + +.options span { + display: block; + padding: 1rem; + width: 100%; + border-radius: 5px; + vertical-align: middle; +} + +.options span:hover { + background-color: var(--1_1); +} + +.selected-multiple-option { + padding: .8rem; + border-radius: 5px; + background-color: var(--bg); + margin: .5rem .3rem; +} + +.selected-multiple-option svg { + width: 1.3rem; + height: 1.3rem; + vertical-align: middle; + margin-bottom: .5rem; + cursor: pointer; +} + +.select-multiple-show { + margin: 1rem 0; + font-size: 1.5rem; +} + +.underlined-text { + font-size: 1.7rem; + cursor: pointer; + margin-bottom: .5rem; + display: block; +} + +.settings .submit-btn { + padding: 1rem 2rem; + font-size: 1.5rem; + background: var(--3); + color: var(--bg); + border-radius: .5rem; + border: 1px solid transparent; + font-weight: bold; + transition: background .5s ease-in; + cursor: pointer; +} + +.settings .submit-btn:hover { + border: 1px solid var(--bg); + background: var(--3_1); + box-shadow: 0px 0px 2px 2px var(--fg); +} + +.settings .submit-btn:active { + outline: none; + border: 2px solid var(--bg); +} + +.errTxt { + color: white; + background: red; + padding: .5rem; + border-radius: 5px; + font-size: 1.3rem; + width: max-content; +} \ No newline at end of file diff --git a/public/templates/navbar.html b/public/templates/navbar.html index f5f581f..87d6550 100644 --- a/public/templates/navbar.html +++ b/public/templates/navbar.html @@ -1,5 +1,6 @@