0
0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-25 07:28:22 -05:00

Compare commits

..

1 Commits

Author SHA1 Message Date
Kekma
089d598db6
Merge 4d5e7bddda into 4847a6eed2 2024-09-03 07:40:40 +00:00
4 changed files with 64 additions and 53 deletions

47
Cargo.lock generated
View File

@ -351,7 +351,7 @@ version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa"
dependencies = [ dependencies = [
"brotli", "brotli 5.0.0",
"flate2", "flate2",
"futures-core", "futures-core",
"memchr", "memchr",
@ -490,7 +490,28 @@ checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b"
dependencies = [ dependencies = [
"alloc-no-stdlib", "alloc-no-stdlib",
"alloc-stdlib", "alloc-stdlib",
"brotli-decompressor", "brotli-decompressor 2.5.1",
]
[[package]]
name = "brotli"
version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor 4.0.0",
]
[[package]]
name = "brotli-decompressor"
version = "4.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
] ]
[[package]] [[package]]
@ -851,7 +872,7 @@ dependencies = [
"clap", "clap",
"criterion-plot", "criterion-plot",
"is-terminal", "is-terminal",
"itertools 0.10.5", "itertools",
"num-traits", "num-traits",
"once_cell", "once_cell",
"oorandom", "oorandom",
@ -870,7 +891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
dependencies = [ dependencies = [
"cast", "cast",
"itertools 0.10.5", "itertools",
] ]
[[package]] [[package]]
@ -1881,15 +1902,6 @@ dependencies = [
"either", "either",
] ]
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]] [[package]]
name = "itoa" name = "itoa"
version = "0.4.8" version = "0.4.8"
@ -1992,7 +2004,7 @@ dependencies = [
"cssparser-color", "cssparser-color",
"data-encoding", "data-encoding",
"getrandom", "getrandom",
"itertools 0.10.5", "itertools",
"lazy_static", "lazy_static",
"parcel_selectors", "parcel_selectors",
"paste", "paste",
@ -2837,9 +2849,9 @@ dependencies = [
[[package]] [[package]]
name = "quinn-proto" name = "quinn-proto"
version = "0.11.8" version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd"
dependencies = [ dependencies = [
"bytes 1.6.0", "bytes 1.6.0",
"rand 0.8.5", "rand 0.8.5",
@ -4449,7 +4461,7 @@ dependencies = [
[[package]] [[package]]
name = "websurfx" name = "websurfx"
version = "1.17.20" version = "1.17.0"
dependencies = [ dependencies = [
"actix-cors", "actix-cors",
"actix-files", "actix-files",
@ -4469,7 +4481,6 @@ dependencies = [
"error-stack", "error-stack",
"fake-useragent", "fake-useragent",
"futures 0.3.30", "futures 0.3.30",
"itertools 0.13.0",
"keyword_extraction", "keyword_extraction",
"lightningcss", "lightningcss",
"log", "log",

View File

@ -1,6 +1,6 @@
[package] [package]
name = "websurfx" name = "websurfx"
version = "1.17.20" version = "1.17.0"
edition = "2021" edition = "2021"
description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
repository = "https://github.com/neon-mmd/websurfx" repository = "https://github.com/neon-mmd/websurfx"
@ -82,13 +82,14 @@ base64 = { version = "0.21.5", default-features = false, features = [
cfg-if = { version = "1.0.0", default-features = false, optional = true } cfg-if = { version = "1.0.0", default-features = false, optional = true }
keyword_extraction = { version = "1.4.3", default-features = false, features = [ keyword_extraction = { version = "1.4.3", default-features = false, features = [
"tf_idf", "tf_idf",
] } ] }
stop-words = { version = "0.8.0", default-features = false, features = ["iso"] } stop-words = { version = "0.8.0", default-features = false, features = ["iso"] }
thesaurus = { version = "0.5.2", default-features = false, optional = true, features = [ thesaurus = { version = "0.5.2", default-features = false, optional = true, features = [
"moby", "moby",
] } ] }
itertools = {version = "0.13.0", default-features = false}
[dev-dependencies] [dev-dependencies]
rusty-hook = { version = "^0.11.2", default-features = false } rusty-hook = { version = "^0.11.2", default-features = false }

View File

@ -4,10 +4,7 @@
use super::error::CacheError; use super::error::CacheError;
use error_stack::Report; use error_stack::Report;
use futures::stream::FuturesUnordered; use futures::stream::FuturesUnordered;
use redis::{ use redis::{aio::ConnectionManager, AsyncCommands, Client, RedisError};
aio::ConnectionManager, AsyncCommands, Client, ExistenceCheck, RedisError, SetExpiry,
SetOptions,
};
/// A constant holding the redis pipeline size. /// A constant holding the redis pipeline size.
const REDIS_PIPELINE_SIZE: usize = 3; const REDIS_PIPELINE_SIZE: usize = 3;
@ -142,14 +139,8 @@ impl RedisCache {
self.current_connection = Default::default(); self.current_connection = Default::default();
for (key, json_result) in keys.zip(json_results) { for (key, json_result) in keys.zip(json_results) {
self.pipeline.set_options( self.pipeline
key, .set_ex(key, json_result, self.cache_ttl.into());
json_result,
SetOptions::default()
.conditional_set(ExistenceCheck::NX)
.get(true)
.with_expiration(SetExpiry::EX(self.cache_ttl.into())),
);
} }
let mut result: Result<(), RedisError> = self let mut result: Result<(), RedisError> = self

View File

@ -12,7 +12,6 @@ use crate::{
results::aggregator::aggregate, results::aggregator::aggregate,
}; };
use actix_web::{get, http::header::ContentType, web, HttpRequest, HttpResponse}; use actix_web::{get, http::header::ContentType, web, HttpRequest, HttpResponse};
use itertools::Itertools;
use regex::Regex; use regex::Regex;
use std::borrow::Cow; use std::borrow::Cow;
use tokio::{ use tokio::{
@ -41,6 +40,7 @@ pub async fn search(
config: web::Data<&'static Config>, config: web::Data<&'static Config>,
cache: web::Data<&'static SharedCache>, cache: web::Data<&'static SharedCache>,
) -> Result<HttpResponse, Box<dyn std::error::Error>> { ) -> Result<HttpResponse, Box<dyn std::error::Error>> {
use std::sync::Arc;
let params = web::Query::<SearchParams>::from_query(req.query_string())?; let params = web::Query::<SearchParams>::from_query(req.query_string())?;
match &params.q { match &params.q {
Some(query) => { Some(query) => {
@ -83,36 +83,44 @@ pub async fn search(
let previous_page = page.saturating_sub(1); let previous_page = page.saturating_sub(1);
let next_page = page + 1; let next_page = page + 1;
let results: (SearchResults, String, bool); let mut results = Arc::new((SearchResults::default(), String::default()));
if page != previous_page { if page != previous_page {
let (previous_results, current_results, next_results) = join!( let (previous_results, current_results, next_results) = join!(
get_results(previous_page), get_results(previous_page),
get_results(page), get_results(page),
get_results(next_page) get_results(next_page)
); );
let (parsed_previous_results, parsed_next_results) =
(previous_results?, next_results?);
results = current_results?; let (cache_keys, results_list) = (
[
parsed_previous_results.1,
results.1.clone(),
parsed_next_results.1,
],
[
parsed_previous_results.0,
results.0.clone(),
parsed_next_results.0,
],
);
let (results_list, cache_keys): (Vec<SearchResults>, Vec<String>) = results = Arc::new(current_results?);
[previous_results?, results.clone(), next_results?]
.into_iter()
.filter_map(|(result, cache_key, flag)| {
dbg!(flag).then_some((result, cache_key))
})
.multiunzip();
tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await }); tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await });
} else { } else {
let (current_results, next_results) = let (current_results, next_results) =
join!(get_results(page), get_results(page + 1)); join!(get_results(page), get_results(page + 1));
results = current_results?; let parsed_next_results = next_results?;
let (results_list, cache_keys): (Vec<SearchResults>, Vec<String>) = results = Arc::new(current_results?);
[results.clone(), next_results?]
.into_iter() let (cache_keys, results_list) = (
.filter_map(|(result, cache_key, flag)| flag.then_some((result, cache_key))) [results.1.clone(), parsed_next_results.1.clone()],
.multiunzip(); [results.0.clone(), parsed_next_results.0],
);
tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await }); tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await });
} }
@ -155,7 +163,7 @@ async fn results(
query: &str, query: &str,
page: u32, page: u32,
search_settings: &server_models::Cookie<'_>, search_settings: &server_models::Cookie<'_>,
) -> Result<(SearchResults, String, bool), Box<dyn std::error::Error>> { ) -> Result<(SearchResults, String), Box<dyn std::error::Error>> {
// eagerly parse cookie value to evaluate safe search level // eagerly parse cookie value to evaluate safe search level
let safe_search_level = search_settings.safe_search_level; let safe_search_level = search_settings.safe_search_level;
@ -174,7 +182,7 @@ async fn results(
// check if fetched cache results was indeed fetched or it was an error and if so // check if fetched cache results was indeed fetched or it was an error and if so
// handle the data accordingly. // handle the data accordingly.
match cached_results { match cached_results {
Ok(results) => Ok((results, cache_key, false)), Ok(results) => Ok((results, cache_key)),
Err(_) => { Err(_) => {
if safe_search_level == 4 { if safe_search_level == 4 {
let mut results: SearchResults = SearchResults::default(); let mut results: SearchResults = SearchResults::default();
@ -188,7 +196,7 @@ async fn results(
.cache_results(&[results.clone()], &[cache_key.clone()]) .cache_results(&[results.clone()], &[cache_key.clone()])
.await?; .await?;
results.set_safe_search_level(safe_search_level); results.set_safe_search_level(safe_search_level);
return Ok((results, cache_key, true)); return Ok((results, cache_key));
} }
} }
@ -227,7 +235,7 @@ async fn results(
.cache_results(&[results.clone()], &[cache_key.clone()]) .cache_results(&[results.clone()], &[cache_key.clone()])
.await?; .await?;
results.set_safe_search_level(safe_search_level); results.set_safe_search_level(safe_search_level);
Ok((results, cache_key, true)) Ok((results, cache_key))
} }
} }
} }