Merge 4d5e7bddda into 4847a6eed2

Cargo.lock

@@ -351,7 +351,7 @@ version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa"
 dependencies = [
- "brotli",
+ "brotli 5.0.0",
  "flate2",
  "futures-core",
  "memchr",
@@ -490,7 +490,28 @@ checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
- "brotli-decompressor",
+ "brotli-decompressor 2.5.1",
+]
+
+[[package]]
+name = "brotli"
+version = "5.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor 4.0.0",
+]
+
+[[package]]
+name = "brotli-decompressor"
+version = "4.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
 ]
 
 [[package]]
@@ -851,7 +872,7 @@ dependencies = [
  "clap",
  "criterion-plot",
  "is-terminal",
- "itertools 0.10.5",
+ "itertools",
  "num-traits",
  "once_cell",
  "oorandom",
@@ -870,7 +891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
 dependencies = [
  "cast",
- "itertools 0.10.5",
+ "itertools",
 ]
 
 [[package]]
@@ -1881,15 +1902,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itoa"
 version = "0.4.8"
@@ -1992,7 +2004,7 @@ dependencies = [
  "cssparser-color",
  "data-encoding",
  "getrandom",
- "itertools 0.10.5",
+ "itertools",
  "lazy_static",
  "parcel_selectors",
  "paste",
@@ -2837,9 +2849,9 @@ dependencies = [
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.8"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6"
+checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd"
 dependencies = [
  "bytes 1.6.0",
  "rand 0.8.5",
@@ -4449,7 +4461,7 @@ dependencies = [
 
 [[package]]
 name = "websurfx"
-version = "1.17.20"
+version = "1.17.0"
 dependencies = [
  "actix-cors",
  "actix-files",
@@ -4469,7 +4481,6 @@ dependencies = [
  "error-stack",
  "fake-useragent",
  "futures 0.3.30",
- "itertools 0.13.0",
  "keyword_extraction",
  "lightningcss",
  "log",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "websurfx"
-version = "1.17.20"
+version = "1.17.0"
 edition = "2021"
 description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
 repository = "https://github.com/neon-mmd/websurfx"
@@ -82,13 +82,14 @@ base64 = { version = "0.21.5", default-features = false, features = [
 cfg-if = { version = "1.0.0", default-features = false, optional = true }
 keyword_extraction = { version = "1.4.3", default-features = false, features = [
     "tf_idf",
+
+
 ] }
 
 stop-words = { version = "0.8.0", default-features = false, features = ["iso"] }
 thesaurus = { version = "0.5.2", default-features = false, optional = true, features = [
     "moby",
-]}
+] }
-itertools = {version = "0.13.0", default-features = false}
 
 [dev-dependencies]
 rusty-hook = { version = "^0.11.2", default-features = false }

src/cache/redis_cacher.rs

@@ -4,10 +4,7 @@
 use super::error::CacheError;
 use error_stack::Report;
 use futures::stream::FuturesUnordered;
-use redis::{
+use redis::{aio::ConnectionManager, AsyncCommands, Client, RedisError};
-    aio::ConnectionManager, AsyncCommands, Client, ExistenceCheck, RedisError, SetExpiry,
-    SetOptions,
-};
 
 /// A constant holding the redis pipeline size.
 const REDIS_PIPELINE_SIZE: usize = 3;
@@ -142,14 +139,8 @@ impl RedisCache {
         self.current_connection = Default::default();
 
         for (key, json_result) in keys.zip(json_results) {
-            self.pipeline.set_options(
+            self.pipeline
-                key,
+                .set_ex(key, json_result, self.cache_ttl.into());
-                json_result,
-                SetOptions::default()
-                    .conditional_set(ExistenceCheck::NX)
-                    .get(true)
-                    .with_expiration(SetExpiry::EX(self.cache_ttl.into())),
-            );
         }
 
         let mut result: Result<(), RedisError> = self

src/server/routes/search.rs

@@ -12,7 +12,6 @@ use crate::{
     results::aggregator::aggregate,
 };
 use actix_web::{get, http::header::ContentType, web, HttpRequest, HttpResponse};
-use itertools::Itertools;
 use regex::Regex;
 use std::borrow::Cow;
 use tokio::{
@@ -41,6 +40,7 @@ pub async fn search(
     config: web::Data<&'static Config>,
     cache: web::Data<&'static SharedCache>,
 ) -> Result<HttpResponse, Box<dyn std::error::Error>> {
+    use std::sync::Arc;
     let params = web::Query::<SearchParams>::from_query(req.query_string())?;
     match &params.q {
         Some(query) => {
@@ -83,36 +83,44 @@ pub async fn search(
             let previous_page = page.saturating_sub(1);
             let next_page = page + 1;
 
-            let results: (SearchResults, String, bool);
+            let mut results = Arc::new((SearchResults::default(), String::default()));
             if page != previous_page {
                 let (previous_results, current_results, next_results) = join!(
                     get_results(previous_page),
                     get_results(page),
                     get_results(next_page)
                 );
+                let (parsed_previous_results, parsed_next_results) =
+                    (previous_results?, next_results?);
 
-                results = current_results?;
+                let (cache_keys, results_list) = (
+                    [
+                        parsed_previous_results.1,
+                        results.1.clone(),
+                        parsed_next_results.1,
+                    ],
+                    [
+                        parsed_previous_results.0,
+                        results.0.clone(),
+                        parsed_next_results.0,
+                    ],
+                );
 
-                let (results_list, cache_keys): (Vec<SearchResults>, Vec<String>) =
+                results = Arc::new(current_results?);
-                    [previous_results?, results.clone(), next_results?]
-                        .into_iter()
-                        .filter_map(|(result, cache_key, flag)| {
-                            dbg!(flag).then_some((result, cache_key))
-                        })
-                        .multiunzip();
 
                 tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await });
             } else {
                 let (current_results, next_results) =
                     join!(get_results(page), get_results(page + 1));
 
-                results = current_results?;
+                let parsed_next_results = next_results?;
 
-                let (results_list, cache_keys): (Vec<SearchResults>, Vec<String>) =
+                results = Arc::new(current_results?);
-                    [results.clone(), next_results?]
+
-                        .into_iter()
+                let (cache_keys, results_list) = (
-                        .filter_map(|(result, cache_key, flag)| flag.then_some((result, cache_key)))
+                    [results.1.clone(), parsed_next_results.1.clone()],
-                        .multiunzip();
+                    [results.0.clone(), parsed_next_results.0],
+                );
 
                 tokio::spawn(async move { cache.cache_results(&results_list, &cache_keys).await });
             }
@@ -155,7 +163,7 @@ async fn results(
     query: &str,
     page: u32,
     search_settings: &server_models::Cookie<'_>,
-) -> Result<(SearchResults, String, bool), Box<dyn std::error::Error>> {
+) -> Result<(SearchResults, String), Box<dyn std::error::Error>> {
     // eagerly parse cookie value to evaluate safe search level
     let safe_search_level = search_settings.safe_search_level;
 
@@ -174,7 +182,7 @@ async fn results(
     // check if fetched cache results was indeed fetched or it was an error and if so
     // handle the data accordingly.
     match cached_results {
-        Ok(results) => Ok((results, cache_key, false)),
+        Ok(results) => Ok((results, cache_key)),
         Err(_) => {
             if safe_search_level == 4 {
                 let mut results: SearchResults = SearchResults::default();
@@ -188,7 +196,7 @@ async fn results(
                         .cache_results(&[results.clone()], &[cache_key.clone()])
                         .await?;
                     results.set_safe_search_level(safe_search_level);
-                    return Ok((results, cache_key, true));
+                    return Ok((results, cache_key));
                 }
             }
 
@@ -227,7 +235,7 @@ async fn results(
                 .cache_results(&[results.clone()], &[cache_key.clone()])
                 .await?;
             results.set_safe_search_level(safe_search_level);
-            Ok((results, cache_key, true))
+            Ok((results, cache_key))
         }
     }
 }