diff --git a/Cargo.lock b/Cargo.lock index be881be..d2f40f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -288,6 +288,12 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + [[package]] name = "askama_escape" version = "0.10.3" @@ -559,7 +565,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ "bytes 1.4.0", + "futures-core", "memchr", + "pin-project-lite", + "tokio 1.32.0", + "tokio-util", ] [[package]] @@ -1049,6 +1059,21 @@ version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.28" @@ -1056,6 +1081,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1070,10 +1096,38 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" dependencies = [ - "futures", + "futures 0.1.31", "num_cpus", ] +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", +] + [[package]] name = "futures-sink" version = "0.3.28" @@ -1092,10 +1146,16 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -1152,7 +1212,7 @@ dependencies = [ "byteorder", "bytes 0.4.12", "fnv", - "futures", + "futures 0.1.31", "http 0.1.21", "indexmap", "log", @@ -1270,7 +1330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "http 0.1.21", "tokio-buf", ] @@ -1317,7 +1377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "futures-cpupool", "h2 0.1.26", "http 0.1.21", @@ -1371,7 +1431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "hyper 0.12.36", "native-tls", "tokio-io", @@ -1721,6 +1781,20 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "mlua" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e" +dependencies = [ + "bstr", + "cc", + "num-traits", + "once_cell", + "pkg-config", + "rustc-hash", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -2075,6 +2149,26 @@ dependencies = [ "siphasher 0.3.11", ] +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -2352,12 +2446,21 @@ version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02" dependencies = [ + "arc-swap", + "async-trait", + "bytes 1.4.0", "combine", + "futures 0.3.28", + "futures-util", "itoa 1.0.9", "percent-encoding 2.3.0", + "pin-project-lite", "ryu", "sha1_smol", "socket2 0.4.9", + "tokio 1.32.0", + "tokio-retry", + "tokio-util", "url 2.4.1", ] @@ -2417,7 +2520,7 @@ dependencies = [ "cookie_store", "encoding_rs", "flate2", - "futures", + "futures 0.1.31", "http 0.1.21", "hyper 0.12.36", "hyper-tls 0.3.2", @@ -2476,36 +2579,18 @@ dependencies = [ "winreg 0.50.0", ] -[[package]] -name = "rlua" -version = "0.19.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d33e5ba15c3d43178f283ed5863d4531e292fc0e56fb773f3bea45f18e3a42a" -dependencies = [ - "bitflags 1.3.2", - "bstr", - "libc", - "num-traits", - "rlua-lua54-sys", -] - -[[package]] -name = "rlua-lua54-sys" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aafabafe1895cb4a2be81a56d7ff3d46bf4b5d2f9cfdbea2ed404cdabe96474" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "rustc-demangle" version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.2.3" @@ -3070,7 +3155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "mio 0.6.23", "num_cpus", "tokio-current-thread", @@ -3109,7 +3194,7 @@ checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46" dependencies = [ "bytes 0.4.12", "either", - "futures", + "futures 0.1.31", ] [[package]] @@ -3118,7 +3203,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" dependencies = [ - "futures", + "futures 0.1.31", "tokio-executor", ] @@ -3129,7 +3214,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" dependencies = [ "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", ] [[package]] @@ -3139,7 +3224,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "log", ] @@ -3171,7 +3256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" dependencies = [ "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", "lazy_static", "log", "mio 0.6.23", @@ -3183,6 +3268,17 @@ dependencies = [ "tokio-sync", ] +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio 1.32.0", +] + [[package]] name = "tokio-sync" version = "0.1.8" @@ -3190,7 +3286,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" dependencies = [ "fnv", - "futures", + "futures 0.1.31", ] [[package]] @@ -3200,7 +3296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" dependencies = [ "bytes 0.4.12", - "futures", + "futures 0.1.31", "iovec", "mio 0.6.23", "tokio-io", @@ -3216,7 +3312,7 @@ dependencies = [ "crossbeam-deque 0.7.4", "crossbeam-queue", "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", "lazy_static", "log", "num_cpus", @@ -3231,7 +3327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" dependencies = [ "crossbeam-utils 0.7.2", - "futures", + "futures 0.1.31", "slab", "tokio-executor", ] @@ -3426,7 +3522,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230" dependencies = [ - "futures", + "futures 0.1.31", "log", "try-lock", ] @@ -3540,15 +3636,16 @@ dependencies = [ "env_logger", "error-stack", "fake-useragent", + "futures 0.3.28", "handlebars", "log", "md5", + "mlua", "once_cell", "rand 0.8.5", "redis", "regex", "reqwest 0.11.20", - "rlua", "rusty-hook", "scraper", "serde", diff --git a/Cargo.toml b/Cargo.toml index ca01dff..b7199b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,26 +7,27 @@ repository = "https://github.com/neon-mmd/websurfx" license = "AGPL-3.0" [dependencies] -reqwest = {version="0.11.19",features=["json"]} +reqwest = {version="0.11.20",features=["json"]} tokio = {version="1.32.0",features=["full"]} -serde = {version="1.0.185",features=["derive"]} +serde = {version="1.0.188",features=["derive"]} handlebars = { version = "4.3.7", features = ["dir_source"] } scraper = {version="0.17.1"} -actix-web = {version="4.3.1", features = ["cookies"]} +actix-web = {version="4.4.0", features = ["cookies"]} actix-files = {version="0.6.2"} actix-cors = {version="0.6.4"} serde_json = {version="1.0.105"} fake-useragent = {version="0.1.3"} env_logger = {version="0.10.0"} log = {version="0.4.20"} -rlua = {version="0.19.7"} -redis = {version="0.23.2"} +mlua = {version="0.8.10",features=["luajit"]} +redis = {version="0.23.2",features=["tokio-comp","connection-manager"]} md5 = {version="0.7.0"} rand={version="0.8.5"} once_cell = {version="1.18.0"} error-stack = {version="0.3.1"} async-trait = {version="0.1.73"} -regex = {version="1.9.3", features=["perf"]} +regex = {version="1.9.4", features=["perf"]} +futures = {version="0.3.28"} [dev-dependencies] rusty-hook = "^0.11.2" diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index 1f26af4..9aa5b69 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -6,7 +6,7 @@ use std::net::TcpListener; use websurfx::{config::parser::Config, run}; -/// A dhat heap memory profiler +/// A dhat heap memory profiler #[cfg(feature = "dhat-heap")] #[global_allocator] static ALLOC: dhat::Alloc = dhat::Alloc; @@ -22,7 +22,7 @@ async fn main() -> std::io::Result<()> { // A dhat heap profiler initialization. #[cfg(feature = "dhat-heap")] let _profiler = dhat::Profiler::new_heap(); - + // Initialize the parsed config file. let config = Config::parse(false).unwrap(); diff --git a/src/lib.rs b/src/lib.rs index cd83d8a..e76344b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,7 +42,7 @@ use handler::paths::{file_path, FileType}; pub fn run(listener: TcpListener, config: Config) -> std::io::Result { let mut handlebars: Handlebars = Handlebars::new(); - let public_folder_path: String = file_path(FileType::Theme)?; + let public_folder_path: &str = file_path(FileType::Theme)?; handlebars .register_templates_directory(".html", format!("{}/templates", public_folder_path)) diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index 3f06ecb..f9bb642 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -71,7 +71,7 @@ pub async fn aggregate( upstream_search_engines: Vec, request_timeout: u8, ) -> Result> { - let user_agent: String = random_user_agent(); + let user_agent: &str = random_user_agent(); // Add a random delay before making the request. if random_delay || !debug { @@ -89,10 +89,9 @@ pub async fn aggregate( let (name, search_engine) = engine_handler.into_name_engine(); names.push(name); let query: String = query.clone(); - let user_agent: String = user_agent.clone(); tasks.push(tokio::spawn(async move { search_engine - .results(query, page, user_agent.clone(), request_timeout) + .results(query, page, user_agent.to_owned(), request_timeout) .await })); } @@ -155,13 +154,13 @@ pub async fn aggregate( filter_with_lists( &mut result_map, &mut blacklist_map, - &file_path(FileType::BlockList)?, + file_path(FileType::BlockList)?, )?; filter_with_lists( &mut blacklist_map, &mut result_map, - &file_path(FileType::AllowList)?, + file_path(FileType::AllowList)?, )?; drop(blacklist_map); diff --git a/src/server/routes.rs b/src/server/routes.rs index 8910f8f..f00cced 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -159,9 +159,9 @@ async fn results( req: HttpRequest, ) -> Result> { //Initialize redis cache connection struct - let mut redis_cache = RedisCache::new(config.redis_url.clone())?; + let mut redis_cache = RedisCache::new(&config.redis_url, 5).await?; // fetch the cached results json. - let cached_results_json = redis_cache.cached_json(&url); + let cached_results_json = redis_cache.cached_json(&url).await; // check if fetched cache results was indeed fetched or it was an error and if so // handle the data accordingly. match cached_results_json { @@ -206,7 +206,9 @@ async fn results( } }; results.add_style(config.style.clone()); - redis_cache.cache_results(serde_json::to_string(&results)?, &url)?; + redis_cache + .cache_results(&serde_json::to_string(&results)?, &url) + .await?; Ok(results) } }