mirror of
https://github.com/neon-mmd/websurfx.git
synced 2024-11-21 21:48:21 -05:00
neon_arch
parent
7fd9246ee4
commit
a799af3125
181
Cargo.lock
generated
181
Cargo.lock
generated
@ -288,6 +288,12 @@ version = "1.0.75"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
|
||||
|
||||
[[package]]
|
||||
name = "arc-swap"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
|
||||
|
||||
[[package]]
|
||||
name = "askama_escape"
|
||||
version = "0.10.3"
|
||||
@ -559,7 +565,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
|
||||
dependencies = [
|
||||
"bytes 1.4.0",
|
||||
"futures-core",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"tokio 1.32.0",
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1049,6 +1059,21 @@ version = "0.1.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-executor",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.28"
|
||||
@ -1056,6 +1081,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1070,10 +1096,38 @@ version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"num_cpus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.28"
|
||||
@ -1092,10 +1146,16 @@ version = "0.3.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-macro",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1152,7 +1212,7 @@ dependencies = [
|
||||
"byteorder",
|
||||
"bytes 0.4.12",
|
||||
"fnv",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"http 0.1.21",
|
||||
"indexmap",
|
||||
"log",
|
||||
@ -1270,7 +1330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"
|
||||
dependencies = [
|
||||
"bytes 0.4.12",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"http 0.1.21",
|
||||
"tokio-buf",
|
||||
]
|
||||
@ -1317,7 +1377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52"
|
||||
dependencies = [
|
||||
"bytes 0.4.12",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"futures-cpupool",
|
||||
"h2 0.1.26",
|
||||
"http 0.1.21",
|
||||
@ -1371,7 +1431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f"
|
||||
dependencies = [
|
||||
"bytes 0.4.12",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"hyper 0.12.36",
|
||||
"native-tls",
|
||||
"tokio-io",
|
||||
@ -1721,6 +1781,20 @@ dependencies = [
|
||||
"ws2_32-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mlua"
|
||||
version = "0.8.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e"
|
||||
dependencies = [
|
||||
"bstr",
|
||||
"cc",
|
||||
"num-traits",
|
||||
"once_cell",
|
||||
"pkg-config",
|
||||
"rustc-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "native-tls"
|
||||
version = "0.2.11"
|
||||
@ -2075,6 +2149,26 @@ dependencies = [
|
||||
"siphasher 0.3.11",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
|
||||
dependencies = [
|
||||
"pin-project-internal",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-internal"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.13"
|
||||
@ -2352,12 +2446,21 @@ version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"bytes 1.4.0",
|
||||
"combine",
|
||||
"futures 0.3.28",
|
||||
"futures-util",
|
||||
"itoa 1.0.9",
|
||||
"percent-encoding 2.3.0",
|
||||
"pin-project-lite",
|
||||
"ryu",
|
||||
"sha1_smol",
|
||||
"socket2 0.4.9",
|
||||
"tokio 1.32.0",
|
||||
"tokio-retry",
|
||||
"tokio-util",
|
||||
"url 2.4.1",
|
||||
]
|
||||
|
||||
@ -2417,7 +2520,7 @@ dependencies = [
|
||||
"cookie_store",
|
||||
"encoding_rs",
|
||||
"flate2",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"http 0.1.21",
|
||||
"hyper 0.12.36",
|
||||
"hyper-tls 0.3.2",
|
||||
@ -2476,36 +2579,18 @@ dependencies = [
|
||||
"winreg 0.50.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rlua"
|
||||
version = "0.19.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d33e5ba15c3d43178f283ed5863d4531e292fc0e56fb773f3bea45f18e3a42a"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"bstr",
|
||||
"libc",
|
||||
"num-traits",
|
||||
"rlua-lua54-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rlua-lua54-sys"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7aafabafe1895cb4a2be81a56d7ff3d46bf4b5d2f9cfdbea2ed404cdabe96474"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.2.3"
|
||||
@ -3070,7 +3155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
|
||||
dependencies = [
|
||||
"bytes 0.4.12",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"mio 0.6.23",
|
||||
"num_cpus",
|
||||
"tokio-current-thread",
|
||||
@ -3109,7 +3194,7 @@ checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46"
|
||||
dependencies = [
|
||||
"bytes 0.4.12",
|
||||
"either",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3118,7 +3203,7 @@ version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"tokio-executor",
|
||||
]
|
||||
|
||||
@ -3129,7 +3214,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671"
|
||||
dependencies = [
|
||||
"crossbeam-utils 0.7.2",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3139,7 +3224,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674"
|
||||
dependencies = [
|
||||
"bytes 0.4.12",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"log",
|
||||
]
|
||||
|
||||
@ -3171,7 +3256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351"
|
||||
dependencies = [
|
||||
"crossbeam-utils 0.7.2",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"mio 0.6.23",
|
||||
@ -3183,6 +3268,17 @@ dependencies = [
|
||||
"tokio-sync",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-retry"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
|
||||
dependencies = [
|
||||
"pin-project",
|
||||
"rand 0.8.5",
|
||||
"tokio 1.32.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-sync"
|
||||
version = "0.1.8"
|
||||
@ -3190,7 +3286,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3200,7 +3296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72"
|
||||
dependencies = [
|
||||
"bytes 0.4.12",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"iovec",
|
||||
"mio 0.6.23",
|
||||
"tokio-io",
|
||||
@ -3216,7 +3312,7 @@ dependencies = [
|
||||
"crossbeam-deque 0.7.4",
|
||||
"crossbeam-queue",
|
||||
"crossbeam-utils 0.7.2",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"num_cpus",
|
||||
@ -3231,7 +3327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296"
|
||||
dependencies = [
|
||||
"crossbeam-utils 0.7.2",
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"slab",
|
||||
"tokio-executor",
|
||||
]
|
||||
@ -3426,7 +3522,7 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"futures 0.1.31",
|
||||
"log",
|
||||
"try-lock",
|
||||
]
|
||||
@ -3540,15 +3636,16 @@ dependencies = [
|
||||
"env_logger",
|
||||
"error-stack",
|
||||
"fake-useragent",
|
||||
"futures 0.3.28",
|
||||
"handlebars",
|
||||
"log",
|
||||
"md5",
|
||||
"mlua",
|
||||
"once_cell",
|
||||
"rand 0.8.5",
|
||||
"redis",
|
||||
"regex",
|
||||
"reqwest 0.11.20",
|
||||
"rlua",
|
||||
"rusty-hook",
|
||||
"scraper",
|
||||
"serde",
|
||||
|
||||
13
Cargo.toml
13
Cargo.toml
@ -7,26 +7,27 @@ repository = "https://github.com/neon-mmd/websurfx"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
[dependencies]
|
||||
reqwest = {version="0.11.19",features=["json"]}
|
||||
reqwest = {version="0.11.20",features=["json"]}
|
||||
tokio = {version="1.32.0",features=["full"]}
|
||||
serde = {version="1.0.185",features=["derive"]}
|
||||
serde = {version="1.0.188",features=["derive"]}
|
||||
handlebars = { version = "4.3.7", features = ["dir_source"] }
|
||||
scraper = {version="0.17.1"}
|
||||
actix-web = {version="4.3.1", features = ["cookies"]}
|
||||
actix-web = {version="4.4.0", features = ["cookies"]}
|
||||
actix-files = {version="0.6.2"}
|
||||
actix-cors = {version="0.6.4"}
|
||||
serde_json = {version="1.0.105"}
|
||||
fake-useragent = {version="0.1.3"}
|
||||
env_logger = {version="0.10.0"}
|
||||
log = {version="0.4.20"}
|
||||
rlua = {version="0.19.7"}
|
||||
redis = {version="0.23.2"}
|
||||
mlua = {version="0.8.10",features=["luajit"]}
|
||||
redis = {version="0.23.2",features=["tokio-comp","connection-manager"]}
|
||||
md5 = {version="0.7.0"}
|
||||
rand={version="0.8.5"}
|
||||
once_cell = {version="1.18.0"}
|
||||
error-stack = {version="0.3.1"}
|
||||
async-trait = {version="0.1.73"}
|
||||
regex = {version="1.9.3", features=["perf"]}
|
||||
regex = {version="1.9.4", features=["perf"]}
|
||||
futures = {version="0.3.28"}
|
||||
|
||||
[dev-dependencies]
|
||||
rusty-hook = "^0.11.2"
|
||||
|
||||
@ -42,7 +42,7 @@ use handler::paths::{file_path, FileType};
|
||||
pub fn run(listener: TcpListener, config: Config) -> std::io::Result<Server> {
|
||||
let mut handlebars: Handlebars = Handlebars::new();
|
||||
|
||||
let public_folder_path: String = file_path(FileType::Theme)?;
|
||||
let public_folder_path: &str = file_path(FileType::Theme)?;
|
||||
|
||||
handlebars
|
||||
.register_templates_directory(".html", format!("{}/templates", public_folder_path))
|
||||
|
||||
@ -71,7 +71,7 @@ pub async fn aggregate(
|
||||
upstream_search_engines: Vec<EngineHandler>,
|
||||
request_timeout: u8,
|
||||
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
||||
let user_agent: String = random_user_agent();
|
||||
let user_agent: &str = random_user_agent();
|
||||
|
||||
// Add a random delay before making the request.
|
||||
if random_delay || !debug {
|
||||
@ -89,10 +89,9 @@ pub async fn aggregate(
|
||||
let (name, search_engine) = engine_handler.into_name_engine();
|
||||
names.push(name);
|
||||
let query: String = query.clone();
|
||||
let user_agent: String = user_agent.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
search_engine
|
||||
.results(query, page, user_agent.clone(), request_timeout)
|
||||
.results(query, page, user_agent.to_owned(), request_timeout)
|
||||
.await
|
||||
}));
|
||||
}
|
||||
@ -155,13 +154,13 @@ pub async fn aggregate(
|
||||
filter_with_lists(
|
||||
&mut result_map,
|
||||
&mut blacklist_map,
|
||||
&file_path(FileType::BlockList)?,
|
||||
file_path(FileType::BlockList)?,
|
||||
)?;
|
||||
|
||||
filter_with_lists(
|
||||
&mut blacklist_map,
|
||||
&mut result_map,
|
||||
&file_path(FileType::AllowList)?,
|
||||
file_path(FileType::AllowList)?,
|
||||
)?;
|
||||
|
||||
drop(blacklist_map);
|
||||
|
||||
@ -159,9 +159,9 @@ async fn results(
|
||||
req: HttpRequest,
|
||||
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
||||
//Initialize redis cache connection struct
|
||||
let mut redis_cache = RedisCache::new(config.redis_url.clone())?;
|
||||
let mut redis_cache = RedisCache::new(&config.redis_url, 5).await?;
|
||||
// fetch the cached results json.
|
||||
let cached_results_json = redis_cache.cached_json(&url);
|
||||
let cached_results_json = redis_cache.cached_json(&url).await;
|
||||
// check if fetched cache results was indeed fetched or it was an error and if so
|
||||
// handle the data accordingly.
|
||||
match cached_results_json {
|
||||
@ -206,7 +206,9 @@ async fn results(
|
||||
}
|
||||
};
|
||||
results.add_style(config.style.clone());
|
||||
redis_cache.cache_results(serde_json::to_string(&results)?, &url)?;
|
||||
redis_cache
|
||||
.cache_results(&serde_json::to_string(&results)?, &url)
|
||||
.await?;
|
||||
Ok(results)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user