From 12bfc5276aa58e67a627f57803f8989e285351d8 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Sun, 10 Sep 2023 18:56:54 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B9=20chore:=20make=20clippy=20happy?= =?UTF-8?q?=20(#201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 90 ++++++++++++++++++++++--------- Cargo.toml | 4 +- src/config/parser.rs | 2 +- src/results/aggregation_models.rs | 4 +- src/results/aggregator.rs | 6 +-- src/server/routes.rs | 18 ++++--- 6 files changed, 84 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5efe8f4..989aa5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -288,6 +288,12 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + [[package]] name = "askama_escape" version = "0.10.3" @@ -565,7 +571,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ "bytes 1.4.0", + "futures-core", "memchr", + "pin-project-lite", + "tokio 1.32.0", + "tokio-util", ] [[package]] @@ -1822,6 +1832,20 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "mlua" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e" +dependencies = [ + "bstr", + "cc", + "num-traits", + "once_cell", + "pkg-config", + "rustc-hash", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -2177,6 +2201,26 @@ dependencies = [ "siphasher 0.3.11", ] +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -2454,12 +2498,21 @@ version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba" dependencies = [ + "arc-swap", + "async-trait", + "bytes 1.4.0", "combine", + "futures 0.3.28", + "futures-util", "itoa 1.0.9", "percent-encoding 2.3.0", + "pin-project-lite", "ryu", "sha1_smol", "socket2 0.4.9", + "tokio 1.32.0", + "tokio-retry", + "tokio-util", "url 2.4.1", ] @@ -2578,30 +2631,6 @@ dependencies = [ "winreg 0.50.0", ] -[[package]] -name = "rlua" -version = "0.19.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d33e5ba15c3d43178f283ed5863d4531e292fc0e56fb773f3bea45f18e3a42a" -dependencies = [ - "bitflags 1.3.2", - "bstr", - "libc", - "num-traits", - "rlua-lua54-sys", -] - -[[package]] -name = "rlua-lua54-sys" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aafabafe1895cb4a2be81a56d7ff3d46bf4b5d2f9cfdbea2ed404cdabe96474" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "rustc-demangle" version = "0.1.23" @@ -3310,6 +3339,17 @@ dependencies = [ "tokio-sync", ] +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand 0.8.5", + "tokio 1.32.0", +] + [[package]] name = "tokio-sync" version = "0.1.8" @@ -3674,12 +3714,12 @@ dependencies = [ "log", "md5", "mimalloc", + "mlua", "once_cell", "rand 0.8.5", "redis", "regex", "reqwest 0.11.20", - "rlua", "rusty-hook", "scraper", "serde", diff --git a/Cargo.toml b/Cargo.toml index 093ec14..cf505fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,8 +19,8 @@ serde_json = {version="1.0.105"} fake-useragent = {version="0.1.3"} env_logger = {version="0.10.0"} log = {version="0.4.20"} -rlua = {version="0.19.7"} -redis = {version="0.23.3"} +mlua = {version="0.8.10", features=["luajit"]} +redis = {version="0.23.3", features=["tokio-comp","connection-manager"]} md5 = {version="0.7.0"} rand={version="0.8.5"} once_cell = {version="1.18.0"} diff --git a/src/config/parser.rs b/src/config/parser.rs index ff29bb2..f5e6d48 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -117,7 +117,7 @@ impl Config { .get::<_, HashMap>("upstream_search_engines")? .into_iter() .filter_map(|(key, value)| value.then_some(key)) - .filter_map(|engine| crate::engines::engine_models::EngineHandler::new(engine)) + .filter_map(|engine| crate::engines::engine_models::EngineHandler::new(&engine)) .collect(), request_timeout: globals.get::<_, u8>("request_timeout")?, threads, diff --git a/src/results/aggregation_models.rs b/src/results/aggregation_models.rs index 6ccd584..280767c 100644 --- a/src/results/aggregation_models.rs +++ b/src/results/aggregation_models.rs @@ -132,9 +132,9 @@ impl SearchResults { ) -> Self { Self { results, - page_query, + page_query: page_query.to_owned(), style: Style::default(), - engine_errors_info, + engine_errors_info: engine_errors_info.to_owned(), disallowed: Default::default(), filtered: Default::default(), } diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index 4452066..734a65f 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -93,7 +93,7 @@ pub async fn aggregate( tasks.push(tokio::spawn(async move { search_engine .results( - query, + &query, page, user_agent.clone(), request_timeout, @@ -162,13 +162,13 @@ pub async fn aggregate( filter_with_lists( &mut result_map, &mut blacklist_map, - &file_path(FileType::BlockList)?, + file_path(FileType::BlockList)?, )?; filter_with_lists( &mut blacklist_map, &mut result_map, - &file_path(FileType::AllowList)?, + file_path(FileType::AllowList)?, )?; drop(blacklist_map); diff --git a/src/server/routes.rs b/src/server/routes.rs index 74f3c92..3d69e78 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -22,7 +22,7 @@ use tokio::join; // ---- Constants ---- /// Initialize redis cache connection once and store it on the heap. -const REDIS_CACHE: async_once_cell::OnceCell = async_once_cell::OnceCell::new(); +static REDIS_CACHE: async_once_cell::OnceCell = async_once_cell::OnceCell::new(); /// A named struct which deserializes all the user provided search parameters and stores them. /// @@ -184,7 +184,7 @@ async fn results( req: HttpRequest, safe_search: u8, ) -> Result> { - let redis_cache: RedisCache = REDIS_CACHE + let mut redis_cache: RedisCache = REDIS_CACHE .get_or_init(async { // Initialize redis cache connection pool only one and store it in the heap. RedisCache::new(&config.redis_url, 5).await.unwrap() @@ -203,14 +203,16 @@ async fn results( if safe_search == 4 { let mut results: SearchResults = SearchResults::default(); let mut _flag: bool = - is_match_from_filter_list(&file_path(FileType::BlockList)?, &query)?; - _flag = !is_match_from_filter_list(&file_path(FileType::AllowList)?, &query)?; + is_match_from_filter_list(file_path(FileType::BlockList)?, query)?; + _flag = !is_match_from_filter_list(file_path(FileType::AllowList)?, query)?; if _flag { results.set_disallowed(); results.add_style(&config.style); - results.set_page_query(&query); - redis_cache.cache_results(serde_json::to_string(&results)?, &url)?; + results.set_page_query(query); + redis_cache + .cache_results(&serde_json::to_string(&results)?, &url) + .await?; return Ok(results); } } @@ -257,7 +259,9 @@ async fn results( results.set_filtered(); } results.add_style(&config.style); - redis_cache.cache_results(serde_json::to_string(&results)?, &url)?; + redis_cache + .cache_results(&serde_json::to_string(&results)?, &url) + .await?; Ok(results) } }