0
0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-22 22:18:23 -05:00

Compare commits

..

1 Commits

Author SHA1 Message Date
Abhinav Pandey
d045154640
Merge f2a32252ce into 2df6499fb2 2024-02-08 04:53:23 +00:00
4 changed files with 8 additions and 12 deletions

View File

@ -30,8 +30,6 @@ pub struct Config {
pub logging: bool, pub logging: bool,
/// It stores the option to whether enable or disable debug mode. /// It stores the option to whether enable or disable debug mode.
pub debug: bool, pub debug: bool,
/// It toggles whether to use adaptive HTTP windows
pub adaptive_window: bool,
/// It stores all the engine names that were enabled by the user. /// It stores all the engine names that were enabled by the user.
pub upstream_search_engines: HashMap<String, bool>, pub upstream_search_engines: HashMap<String, bool>,
/// It stores the time (secs) which controls the server request timeout. /// It stores the time (secs) which controls the server request timeout.
@ -70,7 +68,6 @@ impl Config {
let debug: bool = globals.get::<_, bool>("debug")?; let debug: bool = globals.get::<_, bool>("debug")?;
let logging: bool = globals.get::<_, bool>("logging")?; let logging: bool = globals.get::<_, bool>("logging")?;
let adaptive_window: bool = globals.get::<_, bool>("adaptive_window")?;
if !logging_initialized { if !logging_initialized {
set_logging_level(debug, logging); set_logging_level(debug, logging);
@ -128,7 +125,6 @@ impl Config {
}, },
logging, logging,
debug, debug,
adaptive_window,
upstream_search_engines: globals upstream_search_engines: globals
.get::<_, HashMap<String, bool>>("upstream_search_engines")?, .get::<_, HashMap<String, bool>>("upstream_search_engines")?,
request_timeout: globals.get::<_, u8>("request_timeout")?, request_timeout: globals.get::<_, u8>("request_timeout")?,

View File

@ -2,7 +2,6 @@
//! search engines and then removes duplicate results. //! search engines and then removes duplicate results.
use super::user_agent::random_user_agent; use super::user_agent::random_user_agent;
use crate::config::parser::Config;
use crate::handler::{file_path, FileType}; use crate::handler::{file_path, FileType};
use crate::models::{ use crate::models::{
aggregation_models::{EngineErrorInfo, SearchResult, SearchResults}, aggregation_models::{EngineErrorInfo, SearchResult, SearchResults},
@ -67,17 +66,18 @@ type FutureVec = Vec<JoinHandle<Result<HashMap<String, SearchResult>, Report<Eng
pub async fn aggregate( pub async fn aggregate(
query: &str, query: &str,
page: u32, page: u32,
config: &Config, random_delay: bool,
debug: bool,
upstream_search_engines: &[EngineHandler], upstream_search_engines: &[EngineHandler],
request_timeout: u8,
safe_search: u8, safe_search: u8,
) -> Result<SearchResults, Box<dyn std::error::Error>> { ) -> Result<SearchResults, Box<dyn std::error::Error>> {
let client = CLIENT.get_or_init(|| { let client = CLIENT.get_or_init(|| {
ClientBuilder::new() ClientBuilder::new()
.timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server .timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
.https_only(true) .https_only(true)
.gzip(true) .gzip(true)
.brotli(true) .brotli(true)
.http2_adaptive_window(config.adaptive_window)
.build() .build()
.unwrap() .unwrap()
}); });
@ -85,7 +85,7 @@ pub async fn aggregate(
let user_agent: &str = random_user_agent(); let user_agent: &str = random_user_agent();
// Add a random delay before making the request. // Add a random delay before making the request.
if config.aggregator.random_delay || !config.debug { if random_delay || !debug {
let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.subsec_nanos() as f32; let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.subsec_nanos() as f32;
let delay = ((nanos / 1_0000_0000 as f32).floor() as u64) + 1; let delay = ((nanos / 1_0000_0000 as f32).floor() as u64) + 1;
tokio::time::sleep(Duration::from_secs(delay)).await; tokio::time::sleep(Duration::from_secs(delay)).await;

View File

@ -209,12 +209,14 @@ async fn results(
aggregate( aggregate(
query, query,
page, page,
config, config.aggregator.random_delay,
config.debug,
&search_settings &search_settings
.engines .engines
.iter() .iter()
.filter_map(|engine| EngineHandler::new(engine).ok()) .filter_map(|engine| EngineHandler::new(engine).ok())
.collect::<Vec<EngineHandler>>(), .collect::<Vec<EngineHandler>>(),
config.request_timeout,
safe_search_level, safe_search_level,
) )
.await? .await?

View File

@ -14,8 +14,6 @@ rate_limiter = {
number_of_requests = 20, -- The number of request that are allowed within a provided time limit. number_of_requests = 20, -- The number of request that are allowed within a provided time limit.
time_limit = 3, -- The time limit in which the quantity of requests that should be accepted. time_limit = 3, -- The time limit in which the quantity of requests that should be accepted.
} }
-- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles
https_adaptive_window_size = false
-- ### Search ### -- ### Search ###
-- Filter results based on different levels. The levels provided are: -- Filter results based on different levels. The levels provided are: