From 05899997fd0a311eead3409f8545de5bfe27a416 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 24 Sep 2023 20:58:40 +0300 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=93=9D=20docs:=20add=20a=20navigation?= =?UTF-8?q?=20link=20for=20the=20new=20building=20page=20(#265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/README.md b/docs/README.md index 7232d86..38b486f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,7 @@ - [Instances](./instances.md) - [Installation](./installation.md) +- [Building](./building.md) - [Configuration](./configuration.md) - [Theming](./theming.md) From 1398a18c690eeb7463f33ef4d12364f5dd3ce89c Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 24 Sep 2023 20:59:54 +0300 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9D=20docs:=20add=20new=20building?= =?UTF-8?q?=20page=20explaining=20different=20build=20options=20(#265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/building.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 docs/building.md diff --git a/docs/building.md b/docs/building.md new file mode 100644 index 0000000..54023ae --- /dev/null +++ b/docs/building.md @@ -0,0 +1,59 @@ +# Build Options + +The project provides 4 caching options as conditionally compiled features. This helps reduce the size of the compiled app by only including the code that is necessary for a particular caching option. + +The different caching features provided are as follows: +- No cache +- Redis cache +- In memory cache +- Hybrid cache + +## No Cache + +This feature disables caching for the search engine. This option can drastically reduce binary size but with the cost that subsequent search requests and previous & next page search results are not cached which can make navigating between pages slower. As well as page refreshes of the same page also becomes slower as each refresh has to fetch the results from the upstream search engines. + +To build the app with this option run the following command: + +``` shell +cargo build -r --no-default-features +``` + +Once you have build the app with this option follow the commands listed on the [**Installation**](./installation.md#install-from-source) page of the docs to run the app. + +## Redis Cache + +This feature enables `Redis` caching ability for the search engine. This option allows the search engine to cache the results on the redis server. This feature can be useful for having a dedicated cache server for multiple devices hosted with the `Websurfx` server which can use the one dedicated cache server for hosting their cache on it. But a disadvantage of this solution is that if the `Redis`server is located far away (for example provided by a vps as service) and if it is unavailable or down for some reason then the `Websurfx` server would not be able to function properly or will crash on startup. + +To build the app with this option run the following command: + +``` shell +cargo build -r --no-default-features --features redis-cache +``` + +Once you have build the app with this option follow the commands listed on the [**Installation**](./installation.md#install-from-source) page of the docs to run the app. + +## In Memory Cache + +This feature enables `In Memory` caching soluion within the search engine and it is the default feature provided by the project. This option allows the search engine to cache the results in the memory which can help increase the speed of the fetched cache results and it also has an advantage that it is extremely reliable as all the results are stored in memory within the search engine. Though the disadvantage of this solution are that caching of results is slightly slower than the `redis-cache` solution, it requires a good amount of memory on the system and as such is not ideal for very low memory devices and is highly unscalable. + +To build the app with this option run the following command: + +``` shell +cargo build -r +``` + +Once you have build the app with this option follow the commands listed on the [**Installation**](./installation.md#install-from-source) page of the docs to run the app. + +## Hybrid Cache + +This feature enables the `Hybrid` caching solution for the search engine which provides the advantages of both `In Memory` caching and `Redis` caching and it is an ideal solution if you need a very resiliant and reliable solution for the `Websurfx` which can provide both speed and reliability. Like for example if the `Redis` server becomes unavailable then the search engine switches to `In Memory` caching until the server becomes available again. This solution can be useful for hosting `Websurfx` instance which will be used by hundreds or thousands of users over the world. + +To build the app with this option run the following command: + +``` shell +cargo build -r --features redis-cache +``` + +Once you have build the app with this option follow the commands listed on the [**Installation**](./installation.md#install-from-source) page of the docs to run the app. + +[⬅️ Go back to Home](./README.md) From a5362569daefcae9335f3653786c5d02eb9cfbf5 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 24 Sep 2023 21:00:44 +0300 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=93=9D=20docs:=20update=20install=20f?= =?UTF-8?q?rom=20source=20documentation=20(#265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/installation.md | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 54d4355..9bd4ec0 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -51,8 +51,19 @@ If you want to use the rolling/edge branch, run the following commands instead: ```shell git clone https://github.com/neon-mmd/websurfx.git cd websurfx -cargo build -r +``` + +Once you have changed the directory to the `websurfx` directory then follow the build options listed in the [building docs](./building.md). + +After that run the following command if you have build the app with the `redis-cache` feature: + +``` shell redis-server --port 8082 & +``` + +After that run the following command to start the search engine: + +``` shell ./target/release/websurfx ``` @@ -79,15 +90,30 @@ After that edit the config.lua file located under `websurfx` directory. In the c -- ### General ### logging = true -- an option to enable or disable logs. debug = false -- an option to enable or disable debug mode. -threads = 10 -- the amount of threads that the app will use to run (the value should be greater than 0). +threads = 8 -- the amount of threads that the app will use to run (the value should be greater than 0). -- ### Server ### port = "8080" -- port on which server should be launched -binding_ip_addr = "0.0.0.0" --ip address on the which server should be launched. -production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) +binding_ip = "0.0.0.0" --ip address on the which server should be launched. +production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users (more than one)) -- if production_use is set to true -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. -request_timeout = 60 -- timeout for the search requests sent to the upstream search engines to be fetched (value in seconds). +request_timeout = 30 -- timeout for the search requests sent to the upstream search engines to be fetched (value in seconds). +rate_limiter = { + number_of_requests = 20, -- The number of request that are allowed within a provided time limit. + time_limit = 3, -- The time limit in which the quantity of requests that should be accepted. +} + +-- ### Search ### +-- Filter results based on different levels. The levels provided are: +-- {{ +-- 0 - None +-- 1 - Low +-- 2 - Moderate +-- 3 - High +-- 4 - Aggressive +-- }} +safe_search = 2 -- ### Website ### -- The different colorschemes provided are: @@ -112,7 +138,10 @@ theme = "simple" -- the theme name which should be used for the website redis_url = "redis://redis:6379" -- redis connection url address on which the client should connect on. -- ### Search Engines ### -upstream_search_engines = { DuckDuckGo = true, Searx = false } -- select the upstream search engines from which the results should be fetched. +upstream_search_engines = { + DuckDuckGo = true, + Searx = false, +} -- select the upstream search engines from which the results should be fetched. ``` After this run the following command to deploy the app: From 74d64ce4836a4aea0f88b273299240482a2476a9 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 24 Sep 2023 21:03:18 +0300 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=93=9D=20docs:=20explain=20the=20newl?= =?UTF-8?q?y=20added=20config=20options=20(#265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/configuration.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 665d939..7bfdfc9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -15,6 +15,7 @@ Some of the configuration options provided in the file are stated below. These a - General - Server +- Search - Website - Cache - Search Engines @@ -29,8 +30,21 @@ Some of the configuration options provided in the file are stated below. These a - **port:** Port number on which server should be launched. - **binding_ip_addr:** IP address on the which server should be launched. -- **production_use:** Whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users). If production_use is set to true. There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. This is newly added option and hence is only available in the **edge version**. +- **production_use:** Whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users). If production_use is set to true. There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. - **request_timeout:** Timeout for the search requests sent to the upstream search engines to be fetched (value in seconds). +- **rate_limiter:** The configuration option to configure rate limiting on the search engine website. + +## Search + +- **safe_search:** This option is used to configure the search filtering based on different safe search levels. (value a number between 0 to 4) + +> This option provides 4 levels of search filtering: +> +> - Level 0 - With this level no search filtering occurs. +> - Level 1 - With this level some search filtering occurs. +> - Level 2 - With this level the upstream search engines are restricted to send sensitive contents like NSFW search results, etc. +> - Level 3 - With this level the regex based filter lists is used alongside level 2 to filter more search results that have slipped in or custom results that needs to be filtered using the filter lists. +> - Level 4 - This level is similar to level 3 except in this level the regex based filter lists are used to disallow users to search sensitive or disallowed content. This level could be useful if you are parent or someone who wants to completely disallow their kids or yourself from watching sensitive content. ## Website @@ -61,6 +75,9 @@ Some of the configuration options provided in the file are stated below. These a - **redis_url:** Redis connection url address on which the client should connect on. +> **Note** +> This option can be commented out if you have compiled the app without the `redis-cache` feature. For more information, See [**building**](./building.md). + ## Search Engines - **upstream_search_engines:** Select from the different upstream search engines from which the results should be fetched. From d59d147cd54806403942a1ec031070cd3f866d60 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 24 Sep 2023 21:04:28 +0300 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=93=9D=20docs:=20update=20the=20`Them?= =?UTF-8?q?es`=20section=20(#265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/theming.md | 126 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 2 deletions(-) diff --git a/docs/theming.md b/docs/theming.md index 86e5693..dd46a65 100644 --- a/docs/theming.md +++ b/docs/theming.md @@ -129,6 +129,126 @@ body { .search_bar button:hover { filter: brightness(1.2); } + +.search_area .search_options { + display: flex; + justify-content: space-between; + align-items: center; +} + +.search_area .search_options select { + margin: 0.7rem 0; + width: 20rem; + background-color: var(--color-one); + color: var(--foreground-color); + padding: 1rem 2rem; + border-radius: 0.5rem; + outline: none; + border: none; + text-transform: capitalize; +} + +.search_area .search_options option:hover { + background-color: var(--color-one); +} + +.result_not_found { + display: flex; + flex-direction: column; + font-size: 1.5rem; + color: var(--foreground-color); +} + +.result_not_found p { + margin: 1rem 0; +} + +.result_not_found ul { + margin: 1rem 0; +} + +.result_not_found img { + width: 40rem; +} +``` + +```css +/* styles for the error box */ +.error_box .error_box_toggle_button { + background: var(--foreground-color); +} + +.error_box .dropdown_error_box { + position: absolute; + display: none; + flex-direction: column; + background: var(--background-color); + border-radius: 0; + margin-left: 2rem; + min-height: 20rem; + min-width: 22rem; +} +.error_box .dropdown_error_box.show { + display: flex; +} +.error_box .dropdown_error_box .error_item, +.error_box .dropdown_error_box .no_errors { + display: flex; + align-items: center; + color: var(--foreground-color); + letter-spacing: 0.1rem; + padding: 1rem; + font-size: 1.2rem; +} +.error_box .dropdown_error_box .error_item { + justify-content: space-between; +} +.error_box .dropdown_error_box .no_errors { + min-height: 18rem; + justify-content: center; +} + +.error_box .dropdown_error_box .error_item:hover { + box-shadow: inset 0 0 100px 100px rgba(255, 255, 255, 0.1); +} + +.error_box .error_item .severity_color { + width: 1.2rem; + height: 1.2rem; +} +.results .result_disallowed, +.results .result_filtered, +.results .result_engine_not_selected { + display: flex; + justify-content: center; + align-items: center; + gap: 10rem; + font-size: 2rem; + color: var(--foreground-color); + margin: 0rem 7rem; +} + +.results .result_disallowed .user_query, +.results .result_filtered .user_query, +.results .result_engine_not_selected .user_query { + color: var(--background-color); + font-weight: 300; +} + +.results .result_disallowed img, +.results .result_filtered img, +.results .result_engine_not_selected img { + width: 30rem; +} + +.results .result_disallowed div, +.results .result_filtered div, +.results .result_engine_not_selected div { + display: flex; + flex-direction: column; + gap: 1rem; + line-break: strict; +} ``` ### Styles for the footer and header @@ -499,7 +619,8 @@ This part is only available right now in the **rolling/edge/unstable** version color: var(--foreground-color); } -.settings_container .user_interface select { +.settings_container .user_interface select, +.settings_container .general select { margin: 0.7rem 0; width: 20rem; background-color: var(--background-color); @@ -511,7 +632,8 @@ This part is only available right now in the **rolling/edge/unstable** version text-transform: capitalize; } -.settings_container .user_interface option:hover { +.settings_container .user_interface option:hover, +.settings_container .general option:hover { background-color: var(--color-one); } From da7a0ed854a6d7effc95fc6128c31aaa950898f8 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 24 Sep 2023 21:06:19 +0300 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=9A=80=20chore:=20bump=20the=20app=20?= =?UTF-8?q?version=20(#265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 18 +++++++++--------- Cargo.toml | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f784810..ee8fd4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -215,7 +215,7 @@ dependencies = [ "serde_urlencoded 0.7.1", "smallvec 1.11.1", "socket2 0.5.4", - "time 0.3.28", + "time 0.3.29", "url 2.4.1", ] @@ -650,7 +650,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding 2.3.0", - "time 0.3.28", + "time 0.3.29", "version_check", ] @@ -3397,9 +3397,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ "deranged", "itoa 1.0.9", @@ -3410,15 +3410,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -3932,7 +3932,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.23.1" +version = "0.23.5" dependencies = [ "actix-cors", "actix-files", diff --git a/Cargo.toml b/Cargo.toml index 363b184..d9f386e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.23.1" +version = "0.23.5" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx"