alamin/websurfx
alamin/websurfx
0
0
Fork 0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-21 21:48:21 -05:00
Code Issues Releases Activity

🧹 chore: make clippy happy (#201)

Browse Source
This commit is contained in:
neon_arch 2023-09-10 18:56:54 +03:00
parent ac4e29a93f
commit 12bfc5276a
6 changed files with 84 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
Cargo.lock generated
View File

@ -288,6 +288,12 @@ version = "1.0.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
[[package]]
name = "arc-swap"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
[[package]]
name = "askama_escape"
version = "0.10.3"
@ -565,7 +571,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
dependencies = [
"bytes 1.4.0",
"futures-core",
"memchr",
"pin-project-lite",
"tokio 1.32.0",
"tokio-util",
]
[[package]]
@ -1822,6 +1832,20 @@ dependencies = [
"ws2_32-sys",
]
[[package]]
name = "mlua"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e"
dependencies = [
"bstr",
"cc",
"num-traits",
"once_cell",
"pkg-config",
"rustc-hash",
]
[[package]]
name = "native-tls"
version = "0.2.11"
@ -2177,6 +2201,26 @@ dependencies = [
"siphasher 0.3.11",
]
[[package]]
name = "pin-project"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.33",
"syn 2.0.29",
]
[[package]]
name = "pin-project-lite"
version = "0.2.13"
@ -2454,12 +2498,21 @@ version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
dependencies = [
"arc-swap",
"async-trait",
"bytes 1.4.0",
"combine",
"futures 0.3.28",
"futures-util",
"itoa 1.0.9",
"percent-encoding 2.3.0",
"pin-project-lite",
"ryu",
"sha1_smol",
"socket2 0.4.9",
"tokio 1.32.0",
"tokio-retry",
"tokio-util",
"url 2.4.1",
]
@ -2578,30 +2631,6 @@ dependencies = [
"winreg 0.50.0",
]
[[package]]
name = "rlua"
version = "0.19.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d33e5ba15c3d43178f283ed5863d4531e292fc0e56fb773f3bea45f18e3a42a"
dependencies = [
"bitflags 1.3.2",
"bstr",
"libc",
"num-traits",
"rlua-lua54-sys",
]
[[package]]
name = "rlua-lua54-sys"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7aafabafe1895cb4a2be81a56d7ff3d46bf4b5d2f9cfdbea2ed404cdabe96474"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "rustc-demangle"
version = "0.1.23"
@ -3310,6 +3339,17 @@ dependencies = [
"tokio-sync",
]
[[package]]
name = "tokio-retry"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
dependencies = [
"pin-project",
"rand 0.8.5",
"tokio 1.32.0",
]
[[package]]
name = "tokio-sync"
version = "0.1.8"
@ -3674,12 +3714,12 @@ dependencies = [
"log",
"md5",
"mimalloc",
"mlua",
"once_cell",
"rand 0.8.5",
"redis",
"regex",
"reqwest 0.11.20",
"rlua",
"rusty-hook",
"scraper",
"serde",

4
Cargo.toml
View File

@ -19,8 +19,8 @@ serde_json = {version="1.0.105"}
fake-useragent = {version="0.1.3"}
env_logger = {version="0.10.0"}
log = {version="0.4.20"}
rlua = {version="0.19.7"}
redis = {version="0.23.3"}
mlua = {version="0.8.10", features=["luajit"]}
redis = {version="0.23.3", features=["tokio-comp","connection-manager"]}
md5 = {version="0.7.0"}
rand={version="0.8.5"}
once_cell = {version="1.18.0"}

2
src/config/parser.rs
View File

@ -117,7 +117,7 @@ impl Config {
.get::<_, HashMap<String, bool>>("upstream_search_engines")?
.into_iter()
.filter_map(|(key, value)| value.then_some(key))
.filter_map(|engine| crate::engines::engine_models::EngineHandler::new(engine))
.filter_map(|engine| crate::engines::engine_models::EngineHandler::new(&engine))
.collect(),
request_timeout: globals.get::<_, u8>("request_timeout")?,
threads,

4
src/results/aggregation_models.rs
View File

@ -132,9 +132,9 @@ impl SearchResults {
) -> Self {
Self {
results,
page_query,
page_query: page_query.to_owned(),
style: Style::default(),
engine_errors_info,
engine_errors_info: engine_errors_info.to_owned(),
disallowed: Default::default(),
filtered: Default::default(),
}

6
src/results/aggregator.rs
View File

@ -93,7 +93,7 @@ pub async fn aggregate(
tasks.push(tokio::spawn(async move {
search_engine
.results(
query,
&query,
page,
user_agent.clone(),
request_timeout,
@ -162,13 +162,13 @@ pub async fn aggregate(
filter_with_lists(
&mut result_map,
&mut blacklist_map,
&file_path(FileType::BlockList)?,
file_path(FileType::BlockList)?,
)?;
filter_with_lists(
&mut blacklist_map,
&mut result_map,
&file_path(FileType::AllowList)?,
file_path(FileType::AllowList)?,
)?;
drop(blacklist_map);

18
src/server/routes.rs
View File

@ -22,7 +22,7 @@ use tokio::join;
// ---- Constants ----
/// Initialize redis cache connection once and store it on the heap.
const REDIS_CACHE: async_once_cell::OnceCell<RedisCache> = async_once_cell::OnceCell::new();
static REDIS_CACHE: async_once_cell::OnceCell<RedisCache> = async_once_cell::OnceCell::new();
/// A named struct which deserializes all the user provided search parameters and stores them.
///
@ -184,7 +184,7 @@ async fn results(
req: HttpRequest,
safe_search: u8,
) -> Result<SearchResults, Box<dyn std::error::Error>> {
let redis_cache: RedisCache = REDIS_CACHE
let mut redis_cache: RedisCache = REDIS_CACHE
.get_or_init(async {
// Initialize redis cache connection pool only one and store it in the heap.
RedisCache::new(&config.redis_url, 5).await.unwrap()
@ -203,14 +203,16 @@ async fn results(
if safe_search == 4 {
let mut results: SearchResults = SearchResults::default();
let mut _flag: bool =
is_match_from_filter_list(&file_path(FileType::BlockList)?, &query)?;
_flag = !is_match_from_filter_list(&file_path(FileType::AllowList)?, &query)?;
is_match_from_filter_list(file_path(FileType::BlockList)?, query)?;
_flag = !is_match_from_filter_list(file_path(FileType::AllowList)?, query)?;
if _flag {
results.set_disallowed();
results.add_style(&config.style);
results.set_page_query(&query);
redis_cache.cache_results(serde_json::to_string(&results)?, &url)?;
results.set_page_query(query);
redis_cache
.cache_results(&serde_json::to_string(&results)?, &url)
.await?;
return Ok(results);
}
}
@ -257,7 +259,9 @@ async fn results(
results.set_filtered();
}
results.add_style(&config.style);
redis_cache.cache_results(serde_json::to_string(&results)?, &url)?;
redis_cache
.cache_results(&serde_json::to_string(&results)?, &url)
.await?;
Ok(results)
}
}