alamin/websurfx
alamin/websurfx
0
0
Fork 0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-21 21:48:21 -05:00
Code Issues Releases Activity

🧹 chore: add fixes to code & make rustfmt & clippy happy (#180)(#178)

Browse Source
This commit is contained in:
neon_arch 2023-08-30 18:09:58 +03:00
parent 7fd9246ee4
commit a799af3125
6 changed files with 158 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

181
Cargo.lock generated
View File

@ -288,6 +288,12 @@ version = "1.0.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
[[package]]
name = "arc-swap"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
[[package]]
name = "askama_escape"
version = "0.10.3"
@ -559,7 +565,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
dependencies = [
"bytes 1.4.0",
"futures-core",
"memchr",
"pin-project-lite",
"tokio 1.32.0",
"tokio-util",
]
[[package]]
@ -1049,6 +1059,21 @@ version = "0.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678"
[[package]]
name = "futures"
version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.28"
@ -1056,6 +1081,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
@ -1070,10 +1096,38 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
dependencies = [
"futures",
"futures 0.1.31",
"num_cpus",
]
[[package]]
name = "futures-executor"
version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
[[package]]
name = "futures-macro"
version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.33",
"syn 2.0.29",
]
[[package]]
name = "futures-sink"
version = "0.3.28"
@ -1092,10 +1146,16 @@ version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
"pin-utils",
"slab",
]
[[package]]
@ -1152,7 +1212,7 @@ dependencies = [
"byteorder",
"bytes 0.4.12",
"fnv",
"futures",
"futures 0.1.31",
"http 0.1.21",
"indexmap",
"log",
@ -1270,7 +1330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"
dependencies = [
"bytes 0.4.12",
"futures",
"futures 0.1.31",
"http 0.1.21",
"tokio-buf",
]
@ -1317,7 +1377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52"
dependencies = [
"bytes 0.4.12",
"futures",
"futures 0.1.31",
"futures-cpupool",
"h2 0.1.26",
"http 0.1.21",
@ -1371,7 +1431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f"
dependencies = [
"bytes 0.4.12",
"futures",
"futures 0.1.31",
"hyper 0.12.36",
"native-tls",
"tokio-io",
@ -1721,6 +1781,20 @@ dependencies = [
"ws2_32-sys",
]
[[package]]
name = "mlua"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e"
dependencies = [
"bstr",
"cc",
"num-traits",
"once_cell",
"pkg-config",
"rustc-hash",
]
[[package]]
name = "native-tls"
version = "0.2.11"
@ -2075,6 +2149,26 @@ dependencies = [
"siphasher 0.3.11",
]
[[package]]
name = "pin-project"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [
"proc-macro2 1.0.66",
"quote 1.0.33",
"syn 2.0.29",
]
[[package]]
name = "pin-project-lite"
version = "0.2.13"
@ -2352,12 +2446,21 @@ version = "0.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02"
dependencies = [
"arc-swap",
"async-trait",
"bytes 1.4.0",
"combine",
"futures 0.3.28",
"futures-util",
"itoa 1.0.9",
"percent-encoding 2.3.0",
"pin-project-lite",
"ryu",
"sha1_smol",
"socket2 0.4.9",
"tokio 1.32.0",
"tokio-retry",
"tokio-util",
"url 2.4.1",
]
@ -2417,7 +2520,7 @@ dependencies = [
"cookie_store",
"encoding_rs",
"flate2",
"futures",
"futures 0.1.31",
"http 0.1.21",
"hyper 0.12.36",
"hyper-tls 0.3.2",
@ -2476,36 +2579,18 @@ dependencies = [
"winreg 0.50.0",
]
[[package]]
name = "rlua"
version = "0.19.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d33e5ba15c3d43178f283ed5863d4531e292fc0e56fb773f3bea45f18e3a42a"
dependencies = [
"bitflags 1.3.2",
"bstr",
"libc",
"num-traits",
"rlua-lua54-sys",
]
[[package]]
name = "rlua-lua54-sys"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7aafabafe1895cb4a2be81a56d7ff3d46bf4b5d2f9cfdbea2ed404cdabe96474"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "rustc-demangle"
version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc_version"
version = "0.2.3"
@ -3070,7 +3155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
dependencies = [
"bytes 0.4.12",
"futures",
"futures 0.1.31",
"mio 0.6.23",
"num_cpus",
"tokio-current-thread",
@ -3109,7 +3194,7 @@ checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46"
dependencies = [
"bytes 0.4.12",
"either",
"futures",
"futures 0.1.31",
]
[[package]]
@ -3118,7 +3203,7 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e"
dependencies = [
"futures",
"futures 0.1.31",
"tokio-executor",
]
@ -3129,7 +3214,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671"
dependencies = [
"crossbeam-utils 0.7.2",
"futures",
"futures 0.1.31",
]
[[package]]
@ -3139,7 +3224,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674"
dependencies = [
"bytes 0.4.12",
"futures",
"futures 0.1.31",
"log",
]
@ -3171,7 +3256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351"
dependencies = [
"crossbeam-utils 0.7.2",
"futures",
"futures 0.1.31",
"lazy_static",
"log",
"mio 0.6.23",
@ -3183,6 +3268,17 @@ dependencies = [
"tokio-sync",
]
[[package]]
name = "tokio-retry"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
dependencies = [
"pin-project",
"rand 0.8.5",
"tokio 1.32.0",
]
[[package]]
name = "tokio-sync"
version = "0.1.8"
@ -3190,7 +3286,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee"
dependencies = [
"fnv",
"futures",
"futures 0.1.31",
]
[[package]]
@ -3200,7 +3296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72"
dependencies = [
"bytes 0.4.12",
"futures",
"futures 0.1.31",
"iovec",
"mio 0.6.23",
"tokio-io",
@ -3216,7 +3312,7 @@ dependencies = [
"crossbeam-deque 0.7.4",
"crossbeam-queue",
"crossbeam-utils 0.7.2",
"futures",
"futures 0.1.31",
"lazy_static",
"log",
"num_cpus",
@ -3231,7 +3327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296"
dependencies = [
"crossbeam-utils 0.7.2",
"futures",
"futures 0.1.31",
"slab",
"tokio-executor",
]
@ -3426,7 +3522,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230"
dependencies = [
"futures",
"futures 0.1.31",
"log",
"try-lock",
]
@ -3540,15 +3636,16 @@ dependencies = [
"env_logger",
"error-stack",
"fake-useragent",
"futures 0.3.28",
"handlebars",
"log",
"md5",
"mlua",
"once_cell",
"rand 0.8.5",
"redis",
"regex",
"reqwest 0.11.20",
"rlua",
"rusty-hook",
"scraper",
"serde",

13
Cargo.toml
View File

@ -7,26 +7,27 @@ repository = "https://github.com/neon-mmd/websurfx"
license = "AGPL-3.0"
[dependencies]
reqwest = {version="0.11.19",features=["json"]}
reqwest = {version="0.11.20",features=["json"]}
tokio = {version="1.32.0",features=["full"]}
serde = {version="1.0.185",features=["derive"]}
serde = {version="1.0.188",features=["derive"]}
handlebars = { version = "4.3.7", features = ["dir_source"] }
scraper = {version="0.17.1"}
actix-web = {version="4.3.1", features = ["cookies"]}
actix-web = {version="4.4.0", features = ["cookies"]}
actix-files = {version="0.6.2"}
actix-cors = {version="0.6.4"}
serde_json = {version="1.0.105"}
fake-useragent = {version="0.1.3"}
env_logger = {version="0.10.0"}
log = {version="0.4.20"}
rlua = {version="0.19.7"}
redis = {version="0.23.2"}
mlua = {version="0.8.10",features=["luajit"]}
redis = {version="0.23.2",features=["tokio-comp","connection-manager"]}
md5 = {version="0.7.0"}
rand={version="0.8.5"}
once_cell = {version="1.18.0"}
error-stack = {version="0.3.1"}
async-trait = {version="0.1.73"}
regex = {version="1.9.3", features=["perf"]}
regex = {version="1.9.4", features=["perf"]}
futures = {version="0.3.28"}
[dev-dependencies]
rusty-hook = "^0.11.2"

4
src/bin/websurfx.rs
View File

@ -6,7 +6,7 @@
use std::net::TcpListener;
use websurfx::{config::parser::Config, run};
/// A dhat heap memory profiler
/// A dhat heap memory profiler
#[cfg(feature = "dhat-heap")]
#[global_allocator]
static ALLOC: dhat::Alloc = dhat::Alloc;
@ -22,7 +22,7 @@ async fn main() -> std::io::Result<()> {
// A dhat heap profiler initialization.
#[cfg(feature = "dhat-heap")]
let _profiler = dhat::Profiler::new_heap();
// Initialize the parsed config file.
let config = Config::parse(false).unwrap();

2
src/lib.rs
View File

@ -42,7 +42,7 @@ use handler::paths::{file_path, FileType};
pub fn run(listener: TcpListener, config: Config) -> std::io::Result<Server> {
let mut handlebars: Handlebars = Handlebars::new();
let public_folder_path: String = file_path(FileType::Theme)?;
let public_folder_path: &str = file_path(FileType::Theme)?;
handlebars
.register_templates_directory(".html", format!("{}/templates", public_folder_path))

9
src/results/aggregator.rs
View File

@ -71,7 +71,7 @@ pub async fn aggregate(
upstream_search_engines: Vec<EngineHandler>,
request_timeout: u8,
) -> Result<SearchResults, Box<dyn std::error::Error>> {
let user_agent: String = random_user_agent();
let user_agent: &str = random_user_agent();
// Add a random delay before making the request.
if random_delay || !debug {
@ -89,10 +89,9 @@ pub async fn aggregate(
let (name, search_engine) = engine_handler.into_name_engine();
names.push(name);
let query: String = query.clone();
let user_agent: String = user_agent.clone();
tasks.push(tokio::spawn(async move {
search_engine
.results(query, page, user_agent.clone(), request_timeout)
.results(query, page, user_agent.to_owned(), request_timeout)
.await
}));
}
@ -155,13 +154,13 @@ pub async fn aggregate(
filter_with_lists(
&mut result_map,
&mut blacklist_map,
&file_path(FileType::BlockList)?,
file_path(FileType::BlockList)?,
)?;
filter_with_lists(
&mut blacklist_map,
&mut result_map,
&file_path(FileType::AllowList)?,
file_path(FileType::AllowList)?,
)?;
drop(blacklist_map);

8
src/server/routes.rs
View File

@ -159,9 +159,9 @@ async fn results(
req: HttpRequest,
) -> Result<SearchResults, Box<dyn std::error::Error>> {
//Initialize redis cache connection struct
let mut redis_cache = RedisCache::new(config.redis_url.clone())?;
let mut redis_cache = RedisCache::new(&config.redis_url, 5).await?;
// fetch the cached results json.
let cached_results_json = redis_cache.cached_json(&url);
let cached_results_json = redis_cache.cached_json(&url).await;
// check if fetched cache results was indeed fetched or it was an error and if so
// handle the data accordingly.
match cached_results_json {
@ -206,7 +206,9 @@ async fn results(
}
};
results.add_style(config.style.clone());
redis_cache.cache_results(serde_json::to_string(&results)?, &url)?;
redis_cache
.cache_results(&serde_json::to_string(&results)?, &url)
.await?;
Ok(results)
}
}