diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 7d990f5..fcb0277 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -44,4 +44,4 @@ jobs: delete-branch: true title: 'chore: update contributors-list' body: | - Automated update to `images/contributors_list.svg` + Automated update to `images/contributors_list.svg` \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index ef32619..9af7c18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ dependencies = [ "futures-sink", "memchr", "pin-project-lite", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-util", "tracing", ] @@ -53,7 +53,7 @@ dependencies = [ "actix-service", "actix-utils", "ahash 0.8.3", - "base64 0.21.1", + "base64 0.21.2", "bitflags", "brotli", "bytes 1.4.0", @@ -75,7 +75,7 @@ dependencies = [ "rand 0.8.5", "sha1", "smallvec 1.10.0", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-util", "tracing", "zstd", @@ -87,7 +87,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "syn 1.0.109", ] @@ -111,7 +111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" dependencies = [ "futures-core", - "tokio 1.28.1", + "tokio 1.28.2", ] [[package]] @@ -125,10 +125,10 @@ dependencies = [ "actix-utils", "futures-core", "futures-util", - "mio 0.8.6", + "mio 0.8.7", "num_cpus", "socket2", - "tokio 1.28.1", + "tokio 1.28.2", "tracing", ] @@ -201,8 +201,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" dependencies = [ "actix-router", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -315,9 +315,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1e31e207a6b8fb791a38ea3105e6cb541f55e4d029902d3039a4ad07cc4105" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "bit-set" @@ -605,8 +605,8 @@ dependencies = [ "itoa 1.0.6", "matches", "phf 0.10.1", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "smallvec 1.10.0", "syn 1.0.109", ] @@ -617,7 +617,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "syn 1.0.109", ] @@ -628,8 +628,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -730,8 +730,8 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", "synstructure", ] @@ -971,7 +971,7 @@ dependencies = [ "http 0.2.9", "indexmap", "slab", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-util", "tracing", ] @@ -1035,8 +1035,8 @@ dependencies = [ "log", "mac", "markup5ever 0.11.0", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -1157,7 +1157,7 @@ dependencies = [ "itoa 1.0.6", "pin-project-lite", "socket2", - "tokio 1.28.1", + "tokio 1.28.2", "tower-service", "tracing", "want 0.3.0", @@ -1185,7 +1185,7 @@ dependencies = [ "bytes 1.4.0", "hyper 0.14.26", "native-tls", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-native-tls", ] @@ -1381,12 +1381,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" [[package]] name = "mac" @@ -1513,14 +1510,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "eebffdb73fe72e917997fad08bdbf31ac50b0fa91cec93e69a0662e4264d454c" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -1606,9 +1603,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" [[package]] name = "openssl" @@ -1631,9 +1628,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -1749,9 +1746,9 @@ checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -1863,8 +1860,8 @@ dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", "proc-macro-hack", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -1942,9 +1939,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.58" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" dependencies = [ "unicode-ident", ] @@ -1970,11 +1967,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ - "proc-macro2 1.0.58", + "proc-macro2 1.0.59", ] [[package]] @@ -2213,9 +2210,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.2" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1a59b5d8e97dee33696bf13c5ba8ab85341c002922fba050069326b9c498974" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" dependencies = [ "aho-corasick", "memchr", @@ -2268,7 +2265,7 @@ version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.1", + "base64 0.21.2", "bytes 1.4.0", "encoding_rs", "futures-core", @@ -2289,7 +2286,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.1", - "tokio 1.28.1", + "tokio 1.28.2", "tokio-native-tls", "tower-service", "url 2.3.1", @@ -2495,9 +2492,9 @@ version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2680,8 +2677,8 @@ checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" dependencies = [ "phf_generator 0.7.24", "phf_shared 0.7.24", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "string_cache_shared", ] @@ -2693,8 +2690,8 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", ] [[package]] @@ -2720,19 +2717,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.16" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "unicode-ident", ] @@ -2742,8 +2739,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -2796,9 +2793,9 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2875,14 +2872,14 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.1" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg 1.1.0", "bytes 1.4.0", "libc", - "mio 0.8.6", + "mio 0.8.7", "num_cpus", "parking_lot 0.12.1", "pin-project-lite", @@ -2940,9 +2937,9 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2952,7 +2949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.28.1", + "tokio 1.28.2", ] [[package]] @@ -3037,7 +3034,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.28.1", + "tokio 1.28.2", "tracing", ] @@ -3112,9 +3109,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-normalization" @@ -3260,9 +3257,9 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", "wasm-bindgen-shared", ] @@ -3284,7 +3281,7 @@ version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "wasm-bindgen-macro-support", ] @@ -3294,9 +3291,9 @@ version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ - "proc-macro2 1.0.58", - "quote 1.0.27", - "syn 2.0.16", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3319,7 +3316,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.9.0" +version = "0.11.0" dependencies = [ "actix-files", "actix-web", @@ -3336,7 +3333,7 @@ dependencies = [ "scraper", "serde", "serde_json", - "tokio 1.28.1", + "tokio 1.28.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 76335f8..17aa0a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.9.0" +version = "0.11.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/docs/configuration.md b/docs/configuration.md index bb10ba6..a5391a3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,12 +11,17 @@ If you have built `websurfx` from source then the configuration file will be loc If you have installed `websurfx` using the package manager of your Linux distro then the default configuration file will be located at `/etc/xdg/websurfx/`. You can copy the default config to `~/.config/websurfx/` and make the changes there and rerun the websurfx server. -Some of the configuration options provided in the file are stated below. These are subdivided into three categories: +Some of the configuration options provided in the file are stated below. These are subdivided into the following categories: +- General - Server - Website - Cache +# General + +- **logging:** An option to enable or disable logs. + ## Server - **port:** Port number on which server should be launched. diff --git a/docs/installation.md b/docs/installation.md index d8f301b..1a04254 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -76,11 +76,17 @@ cd websurfx After that edit the config.lua file located under `websurfx` directory. In the config file you will specifically need to change to values which is `binding_ip_addr` and `redis_connection_url` which should make the config look something like this: ```lua --- Server +-- ### General ### +logging = true -- an option to enable or disable logs. + +-- ### Server ### port = "8080" -- port on which server should be launched binding_ip_addr = "0.0.0.0" --ip address on the which server should be launched. +production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) +-- if production_use is set to true +-- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. --- Website +-- ### Website ### -- The different colorschemes provided are: -- {{ -- catppuccin-mocha @@ -95,12 +101,8 @@ binding_ip_addr = "0.0.0.0" --ip address on the which server should be launched. colorscheme = "catppuccin-mocha" -- the colorscheme name which should be used for the website theme theme = "simple" -- the theme name which should be used for the website --- Caching -redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. - -production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) --- if production_use is set to true - -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. +-- ### Caching ### +redis_connection_url = "redis://redis:6379" -- redis connection url address on which the client should connect on. ``` After this run the following command to deploy the app: diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index ca05713..8661725 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -19,7 +19,7 @@ async fn main() -> std::io::Result<()> { let config = Config::parse().unwrap(); // Initializing logging middleware with level set to default or info. - if config.logging { + if config.logging || config.debug { use env_logger::Env; env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); } diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index dd92f1b..ac200cd 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -26,6 +26,7 @@ pub struct Config { pub redis_connection_url: String, pub aggregator: AggreatorConfig, pub logging: bool, + pub debug: bool, } /// Configuration options for the aggregator. @@ -73,6 +74,7 @@ impl Config { redis_connection_url: globals.get::<_, String>("redis_connection_url")?, aggregator: aggregator_config, logging: globals.get::<_, bool>("logging")?, + debug: globals.get::<_, bool>("debug")?, }) }) } diff --git a/src/search_results_handler/aggregator.rs b/src/search_results_handler/aggregator.rs index 8b86972..8b6bae3 100644 --- a/src/search_results_handler/aggregator.rs +++ b/src/search_results_handler/aggregator.rs @@ -40,12 +40,13 @@ pub async fn aggregate( query: &str, page: u32, random_delay: bool, + debug: bool, ) -> Result> { let user_agent: String = random_user_agent(); let mut result_map: HashMap = HashMap::new(); // Add a random delay before making the request. - if random_delay { + if random_delay || !debug { let mut rng = rand::thread_rng(); let delay_secs = rng.gen_range(1..10); std::thread::sleep(Duration::from_secs(delay_secs)); diff --git a/src/server/routes.rs b/src/server/routes.rs index ead1612..9234d8d 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -128,7 +128,7 @@ pub async fn search( } Err(_) => { let mut results_json: crate::search_results_handler::aggregation_models::SearchResults = - aggregate(query, page, config.aggregator.random_delay).await?; + aggregate(query, page, config.aggregator.random_delay, config.debug).await?; results_json.add_style(config.style.clone()); redis_cache .cache_results(serde_json::to_string(&results_json)?, &page_url)?; diff --git a/websurfx/config.lua b/websurfx/config.lua index 29c4fff..3daaa91 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -1,5 +1,6 @@ -- ### General ### logging = true -- an option to enable or disable logs. +debug = false -- an option to enable or disable debug mode. -- ### Server ### port = "8080" -- port on which server should be launched