0
0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-24 23:18:22 -05:00

Merge branch 'rolling' into contributors-list-automation-ci

This commit is contained in:
neon_arch 2023-05-30 10:08:26 +00:00 committed by GitHub
commit 8839c5518e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 109 additions and 101 deletions

171
Cargo.lock generated
View File

@ -14,7 +14,7 @@ dependencies = [
"futures-sink",
"memchr",
"pin-project-lite",
"tokio 1.28.1",
"tokio 1.28.2",
"tokio-util",
"tracing",
]
@ -53,7 +53,7 @@ dependencies = [
"actix-service",
"actix-utils",
"ahash 0.8.3",
"base64 0.21.1",
"base64 0.21.2",
"bitflags",
"brotli",
"bytes 1.4.0",
@ -75,7 +75,7 @@ dependencies = [
"rand 0.8.5",
"sha1",
"smallvec 1.10.0",
"tokio 1.28.1",
"tokio 1.28.2",
"tokio-util",
"tracing",
"zstd",
@ -87,7 +87,7 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6"
dependencies = [
"quote 1.0.27",
"quote 1.0.28",
"syn 1.0.109",
]
@ -111,7 +111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e"
dependencies = [
"futures-core",
"tokio 1.28.1",
"tokio 1.28.2",
]
[[package]]
@ -125,10 +125,10 @@ dependencies = [
"actix-utils",
"futures-core",
"futures-util",
"mio 0.8.6",
"mio 0.8.7",
"num_cpus",
"socket2",
"tokio 1.28.1",
"tokio 1.28.2",
"tracing",
]
@ -201,8 +201,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9"
dependencies = [
"actix-router",
"proc-macro2 1.0.58",
"quote 1.0.27",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 1.0.109",
]
@ -315,9 +315,9 @@ dependencies = [
[[package]]
name = "base64"
version = "0.21.1"
version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f1e31e207a6b8fb791a38ea3105e6cb541f55e4d029902d3039a4ad07cc4105"
checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
[[package]]
name = "bit-set"
@ -605,8 +605,8 @@ dependencies = [
"itoa 1.0.6",
"matches",
"phf 0.10.1",
"proc-macro2 1.0.58",
"quote 1.0.27",
"proc-macro2 1.0.59",
"quote 1.0.28",
"smallvec 1.10.0",
"syn 1.0.109",
]
@ -617,7 +617,7 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e"
dependencies = [
"quote 1.0.27",
"quote 1.0.28",
"syn 1.0.109",
]
@ -628,8 +628,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
dependencies = [
"convert_case",
"proc-macro2 1.0.58",
"quote 1.0.27",
"proc-macro2 1.0.59",
"quote 1.0.28",
"rustc_version 0.4.0",
"syn 1.0.109",
]
@ -730,8 +730,8 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
dependencies = [
"proc-macro2 1.0.58",
"quote 1.0.27",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 1.0.109",
"synstructure",
]
@ -971,7 +971,7 @@ dependencies = [
"http 0.2.9",
"indexmap",
"slab",
"tokio 1.28.1",
"tokio 1.28.2",
"tokio-util",
"tracing",
]
@ -1035,8 +1035,8 @@ dependencies = [
"log",
"mac",
"markup5ever 0.11.0",
"proc-macro2 1.0.58",
"quote 1.0.27",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 1.0.109",
]
@ -1157,7 +1157,7 @@ dependencies = [
"itoa 1.0.6",
"pin-project-lite",
"socket2",
"tokio 1.28.1",
"tokio 1.28.2",
"tower-service",
"tracing",
"want 0.3.0",
@ -1185,7 +1185,7 @@ dependencies = [
"bytes 1.4.0",
"hyper 0.14.26",
"native-tls",
"tokio 1.28.1",
"tokio 1.28.2",
"tokio-native-tls",
]
@ -1381,12 +1381,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.17"
version = "0.4.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if 1.0.0",
]
checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de"
[[package]]
name = "mac"
@ -1513,14 +1510,14 @@ dependencies = [
[[package]]
name = "mio"
version = "0.8.6"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9"
checksum = "eebffdb73fe72e917997fad08bdbf31ac50b0fa91cec93e69a0662e4264d454c"
dependencies = [
"libc",
"log",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys 0.45.0",
"windows-sys 0.48.0",
]
[[package]]
@ -1606,9 +1603,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.17.1"
version = "1.17.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b"
[[package]]
name = "openssl"
@ -1631,9 +1628,9 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2 1.0.58",
"quote 1.0.27",
"syn 2.0.16",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 2.0.18",
]
[[package]]
@ -1749,9 +1746,9 @@ checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e"
dependencies = [
"pest",
"pest_meta",
"proc-macro2 1.0.58",
"quote 1.0.27",
"syn 2.0.16",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 2.0.18",
]
[[package]]
@ -1863,8 +1860,8 @@ dependencies = [
"phf_generator 0.10.0",
"phf_shared 0.10.0",
"proc-macro-hack",
"proc-macro2 1.0.58",
"quote 1.0.27",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 1.0.109",
]
@ -1942,9 +1939,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.58"
version = "1.0.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8"
checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b"
dependencies = [
"unicode-ident",
]
@ -1970,11 +1967,11 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.27"
version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500"
checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488"
dependencies = [
"proc-macro2 1.0.58",
"proc-macro2 1.0.59",
]
[[package]]
@ -2213,9 +2210,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.8.2"
version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1a59b5d8e97dee33696bf13c5ba8ab85341c002922fba050069326b9c498974"
checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390"
dependencies = [
"aho-corasick",
"memchr",
@ -2268,7 +2265,7 @@ version = "0.11.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55"
dependencies = [
"base64 0.21.1",
"base64 0.21.2",
"bytes 1.4.0",
"encoding_rs",
"futures-core",
@ -2289,7 +2286,7 @@ dependencies = [
"serde",
"serde_json",
"serde_urlencoded 0.7.1",
"tokio 1.28.1",
"tokio 1.28.2",
"tokio-native-tls",
"tower-service",
"url 2.3.1",
@ -2495,9 +2492,9 @@ version = "1.0.163"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e"
dependencies = [
"proc-macro2 1.0.58",
"quote 1.0.27",
"syn 2.0.16",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 2.0.18",
]
[[package]]
@ -2680,8 +2677,8 @@ checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6"
dependencies = [
"phf_generator 0.7.24",
"phf_shared 0.7.24",
"proc-macro2 1.0.58",
"quote 1.0.27",
"proc-macro2 1.0.59",
"quote 1.0.28",
"string_cache_shared",
]
@ -2693,8 +2690,8 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"
dependencies = [
"phf_generator 0.10.0",
"phf_shared 0.10.0",
"proc-macro2 1.0.58",
"quote 1.0.27",
"proc-macro2 1.0.59",
"quote 1.0.28",
]
[[package]]
@ -2720,19 +2717,19 @@ version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2 1.0.58",
"quote 1.0.27",
"proc-macro2 1.0.59",
"quote 1.0.28",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.16"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01"
checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e"
dependencies = [
"proc-macro2 1.0.58",
"quote 1.0.27",
"proc-macro2 1.0.59",
"quote 1.0.28",
"unicode-ident",
]
@ -2742,8 +2739,8 @@ version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
dependencies = [
"proc-macro2 1.0.58",
"quote 1.0.27",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 1.0.109",
"unicode-xid 0.2.4",
]
@ -2796,9 +2793,9 @@ version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
dependencies = [
"proc-macro2 1.0.58",
"quote 1.0.27",
"syn 2.0.16",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 2.0.18",
]
[[package]]
@ -2875,14 +2872,14 @@ dependencies = [
[[package]]
name = "tokio"
version = "1.28.1"
version = "1.28.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105"
checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2"
dependencies = [
"autocfg 1.1.0",
"bytes 1.4.0",
"libc",
"mio 0.8.6",
"mio 0.8.7",
"num_cpus",
"parking_lot 0.12.1",
"pin-project-lite",
@ -2940,9 +2937,9 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2 1.0.58",
"quote 1.0.27",
"syn 2.0.16",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 2.0.18",
]
[[package]]
@ -2952,7 +2949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
dependencies = [
"native-tls",
"tokio 1.28.1",
"tokio 1.28.2",
]
[[package]]
@ -3037,7 +3034,7 @@ dependencies = [
"futures-core",
"futures-sink",
"pin-project-lite",
"tokio 1.28.1",
"tokio 1.28.2",
"tracing",
]
@ -3112,9 +3109,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
[[package]]
name = "unicode-ident"
version = "1.0.8"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0"
[[package]]
name = "unicode-normalization"
@ -3260,9 +3257,9 @@ dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2 1.0.58",
"quote 1.0.27",
"syn 2.0.16",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 2.0.18",
"wasm-bindgen-shared",
]
@ -3284,7 +3281,7 @@ version = "0.2.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258"
dependencies = [
"quote 1.0.27",
"quote 1.0.28",
"wasm-bindgen-macro-support",
]
@ -3294,9 +3291,9 @@ version = "0.2.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8"
dependencies = [
"proc-macro2 1.0.58",
"quote 1.0.27",
"syn 2.0.16",
"proc-macro2 1.0.59",
"quote 1.0.28",
"syn 2.0.18",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@ -3319,7 +3316,7 @@ dependencies = [
[[package]]
name = "websurfx"
version = "0.9.0"
version = "0.11.0"
dependencies = [
"actix-files",
"actix-web",
@ -3336,7 +3333,7 @@ dependencies = [
"scraper",
"serde",
"serde_json",
"tokio 1.28.1",
"tokio 1.28.2",
]
[[package]]

View File

@ -1,6 +1,6 @@
[package]
name = "websurfx"
version = "0.9.0"
version = "0.11.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

View File

@ -11,12 +11,17 @@ If you have built `websurfx` from source then the configuration file will be loc
If you have installed `websurfx` using the package manager of your Linux distro then the default configuration file will be located at `/etc/xdg/websurfx/`. You can copy the default config to `~/.config/websurfx/` and make the changes there and rerun the websurfx server.
Some of the configuration options provided in the file are stated below. These are subdivided into three categories:
Some of the configuration options provided in the file are stated below. These are subdivided into the following categories:
- General
- Server
- Website
- Cache
# General
- **logging:** An option to enable or disable logs.
## Server
- **port:** Port number on which server should be launched.

View File

@ -76,11 +76,17 @@ cd websurfx
After that edit the config.lua file located under `websurfx` directory. In the config file you will specifically need to change to values which is `binding_ip_addr` and `redis_connection_url` which should make the config look something like this:
```lua
-- Server
-- ### General ###
logging = true -- an option to enable or disable logs.
-- ### Server ###
port = "8080" -- port on which server should be launched
binding_ip_addr = "0.0.0.0" --ip address on the which server should be launched.
production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users)
-- if production_use is set to true
-- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.
-- Website
-- ### Website ###
-- The different colorschemes provided are:
-- {{
-- catppuccin-mocha
@ -95,12 +101,8 @@ binding_ip_addr = "0.0.0.0" --ip address on the which server should be launched.
colorscheme = "catppuccin-mocha" -- the colorscheme name which should be used for the website theme
theme = "simple" -- the theme name which should be used for the website
-- Caching
redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.
production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users)
-- if production_use is set to true
-- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.
-- ### Caching ###
redis_connection_url = "redis://redis:6379" -- redis connection url address on which the client should connect on.
```
After this run the following command to deploy the app:

View File

@ -19,7 +19,7 @@ async fn main() -> std::io::Result<()> {
let config = Config::parse().unwrap();
// Initializing logging middleware with level set to default or info.
if config.logging {
if config.logging || config.debug {
use env_logger::Env;
env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
}

View File

@ -26,6 +26,7 @@ pub struct Config {
pub redis_connection_url: String,
pub aggregator: AggreatorConfig,
pub logging: bool,
pub debug: bool,
}
/// Configuration options for the aggregator.
@ -73,6 +74,7 @@ impl Config {
redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
aggregator: aggregator_config,
logging: globals.get::<_, bool>("logging")?,
debug: globals.get::<_, bool>("debug")?,
})
})
}

View File

@ -40,12 +40,13 @@ pub async fn aggregate(
query: &str,
page: u32,
random_delay: bool,
debug: bool,
) -> Result<SearchResults, Box<dyn std::error::Error>> {
let user_agent: String = random_user_agent();
let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
// Add a random delay before making the request.
if random_delay {
if random_delay || !debug {
let mut rng = rand::thread_rng();
let delay_secs = rng.gen_range(1..10);
std::thread::sleep(Duration::from_secs(delay_secs));

View File

@ -128,7 +128,7 @@ pub async fn search(
}
Err(_) => {
let mut results_json: crate::search_results_handler::aggregation_models::SearchResults =
aggregate(query, page, config.aggregator.random_delay).await?;
aggregate(query, page, config.aggregator.random_delay, config.debug).await?;
results_json.add_style(config.style.clone());
redis_cache
.cache_results(serde_json::to_string(&results_json)?, &page_url)?;

View File

@ -1,5 +1,6 @@
-- ### General ###
logging = true -- an option to enable or disable logs.
debug = false -- an option to enable or disable debug mode.
-- ### Server ###
port = "8080" -- port on which server should be launched