mirror of
https://github.com/neon-mmd/websurfx.git
synced 2024-11-22 14:08:23 -05:00
Compare commits
1 Commits
edef4424b1
...
d045154640
Author | SHA1 | Date | |
---|---|---|---|
|
d045154640 |
@ -30,8 +30,6 @@ pub struct Config {
|
||||
pub logging: bool,
|
||||
/// It stores the option to whether enable or disable debug mode.
|
||||
pub debug: bool,
|
||||
/// It toggles whether to use adaptive HTTP windows
|
||||
pub adaptive_window: bool,
|
||||
/// It stores all the engine names that were enabled by the user.
|
||||
pub upstream_search_engines: HashMap<String, bool>,
|
||||
/// It stores the time (secs) which controls the server request timeout.
|
||||
@ -70,7 +68,6 @@ impl Config {
|
||||
|
||||
let debug: bool = globals.get::<_, bool>("debug")?;
|
||||
let logging: bool = globals.get::<_, bool>("logging")?;
|
||||
let adaptive_window: bool = globals.get::<_, bool>("adaptive_window")?;
|
||||
|
||||
if !logging_initialized {
|
||||
set_logging_level(debug, logging);
|
||||
@ -128,7 +125,6 @@ impl Config {
|
||||
},
|
||||
logging,
|
||||
debug,
|
||||
adaptive_window,
|
||||
upstream_search_engines: globals
|
||||
.get::<_, HashMap<String, bool>>("upstream_search_engines")?,
|
||||
request_timeout: globals.get::<_, u8>("request_timeout")?,
|
||||
|
@ -2,7 +2,6 @@
|
||||
//! search engines and then removes duplicate results.
|
||||
|
||||
use super::user_agent::random_user_agent;
|
||||
use crate::config::parser::Config;
|
||||
use crate::handler::{file_path, FileType};
|
||||
use crate::models::{
|
||||
aggregation_models::{EngineErrorInfo, SearchResult, SearchResults},
|
||||
@ -67,17 +66,18 @@ type FutureVec = Vec<JoinHandle<Result<HashMap<String, SearchResult>, Report<Eng
|
||||
pub async fn aggregate(
|
||||
query: &str,
|
||||
page: u32,
|
||||
config: &Config,
|
||||
random_delay: bool,
|
||||
debug: bool,
|
||||
upstream_search_engines: &[EngineHandler],
|
||||
request_timeout: u8,
|
||||
safe_search: u8,
|
||||
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
||||
let client = CLIENT.get_or_init(|| {
|
||||
ClientBuilder::new()
|
||||
.timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
|
||||
.timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
|
||||
.https_only(true)
|
||||
.gzip(true)
|
||||
.brotli(true)
|
||||
.http2_adaptive_window(config.adaptive_window)
|
||||
.build()
|
||||
.unwrap()
|
||||
});
|
||||
@ -85,7 +85,7 @@ pub async fn aggregate(
|
||||
let user_agent: &str = random_user_agent();
|
||||
|
||||
// Add a random delay before making the request.
|
||||
if config.aggregator.random_delay || !config.debug {
|
||||
if random_delay || !debug {
|
||||
let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.subsec_nanos() as f32;
|
||||
let delay = ((nanos / 1_0000_0000 as f32).floor() as u64) + 1;
|
||||
tokio::time::sleep(Duration::from_secs(delay)).await;
|
||||
|
@ -209,12 +209,14 @@ async fn results(
|
||||
aggregate(
|
||||
query,
|
||||
page,
|
||||
config,
|
||||
config.aggregator.random_delay,
|
||||
config.debug,
|
||||
&search_settings
|
||||
.engines
|
||||
.iter()
|
||||
.filter_map(|engine| EngineHandler::new(engine).ok())
|
||||
.collect::<Vec<EngineHandler>>(),
|
||||
config.request_timeout,
|
||||
safe_search_level,
|
||||
)
|
||||
.await?
|
||||
|
@ -14,8 +14,6 @@ rate_limiter = {
|
||||
number_of_requests = 20, -- The number of request that are allowed within a provided time limit.
|
||||
time_limit = 3, -- The time limit in which the quantity of requests that should be accepted.
|
||||
}
|
||||
-- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles
|
||||
https_adaptive_window_size = false
|
||||
|
||||
-- ### Search ###
|
||||
-- Filter results based on different levels. The levels provided are:
|
||||
|
Loading…
Reference in New Issue
Block a user