alamin/websurfx
alamin/websurfx
0
0
Fork 0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-22 05:58:21 -05:00
Code Issues Releases Activity

Merge branch 'rolling' into proxy

Browse Source
This commit is contained in:
neon_arch 2024-10-01 16:02:56 +03:00 committed by GitHub
commit c07d9280be
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 25 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/config/parser.rs
View File

@ -49,9 +49,10 @@ pub struct Config {
pub tcp_connection_keep_alive: u8, pub tcp_connection_keep_alive: u8,
/// It stores the pool idle connection timeout in seconds. /// It stores the pool idle connection timeout in seconds.
pub pool_idle_connection_timeout: u8, pub pool_idle_connection_timeout: u8,
/// Url of the proxy to use for outgoing requests. /// Url of the proxy to use for outgoing requests.
pub proxy: Option<Proxy>, pub proxy: Option<Proxy>,
/// It stores the number of https connections to keep in the pool.
pub number_of_https_connections: u8,
} }
impl Config { impl Config {
@ -152,6 +153,7 @@ impl Config {
request_timeout: globals.get::<_, u8>("request_timeout")?, request_timeout: globals.get::<_, u8>("request_timeout")?,
tcp_connection_keep_alive: globals.get::<_, u8>("tcp_connection_keep_alive")?, tcp_connection_keep_alive: globals.get::<_, u8>("tcp_connection_keep_alive")?,
pool_idle_connection_timeout: globals.get::<_, u8>("pool_idle_connection_timeout")?, pool_idle_connection_timeout: globals.get::<_, u8>("pool_idle_connection_timeout")?,
number_of_https_connections: globals.get::<_, u8>("number_of_https_connections")?,
threads, threads,
client_connection_keep_alive: globals.get::<_, u8>("client_connection_keep_alive")?, client_connection_keep_alive: globals.get::<_, u8>("client_connection_keep_alive")?,
rate_limiter: RateLimiter { rate_limiter: RateLimiter {

2
src/models/aggregation_models.rs
View File

@ -11,7 +11,9 @@ use thesaurus::synonyms;
/// A named struct to store the raw scraped search results scraped search results from the /// A named struct to store the raw scraped search results scraped search results from the
/// upstream search engines before aggregating it.It derives the Clone trait which is needed /// upstream search engines before aggregating it.It derives the Clone trait which is needed
/// to write idiomatic rust using `Iterators`. /// to write idiomatic rust using `Iterators`.
///
/// (href url in html in simple words). /// (href url in html in simple words).
///
#[derive(Clone, Serialize, Deserialize)] #[derive(Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct SearchResult { pub struct SearchResult {

1
src/results/aggregator.rs
View File

@ -81,6 +81,7 @@ pub async fn aggregate(
config.pool_idle_connection_timeout as u64, config.pool_idle_connection_timeout as u64,
)) ))
.tcp_keepalive(Duration::from_secs(config.tcp_connection_keep_alive as u64)) .tcp_keepalive(Duration::from_secs(config.tcp_connection_keep_alive as u64))
.pool_max_idle_per_host(config.number_of_https_connections as usize)
.connect_timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server .connect_timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
.https_only(true) .https_only(true)
.gzip(true) .gzip(true)

2
websurfx/config.lua
View File

@ -18,6 +18,8 @@ rate_limiter = {
} }
-- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles -- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles
https_adaptive_window_size = false https_adaptive_window_size = false
number_of_https_connections = 10 -- the number of https connections that should be available in the connection pool.
-- Set keep-alive timer in seconds; keeps clients connected to the HTTP server, different from the connection to upstream search engines -- Set keep-alive timer in seconds; keeps clients connected to the HTTP server, different from the connection to upstream search engines
client_connection_keep_alive = 120 client_connection_keep_alive = 120