mirror of
https://github.com/neon-mmd/websurfx.git
synced 2024-10-18 06:22:53 -04:00
Merge branch 'rolling' into qwant
This commit is contained in:
mergify[bot]
GitHub
commit
4857dcb976
52
Cargo.lock
generated
52
Cargo.lock
generated
@ -59,9 +59,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "actix-governor"
|
||||
version = "0.5.0"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2e7b88f3804e01bd4191fdb08650430bbfcb43d3d9b2890064df3551ec7d25b"
|
||||
checksum = "0954b0f27aabd8f56bb03f2a77b412ddf3f8c034a3c27b2086c1fc75415760df"
|
||||
dependencies = [
|
||||
"actix-http",
|
||||
"actix-web",
|
||||
@ -1180,9 +1180,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "error-stack"
|
||||
version = "0.4.1"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27a72baa257b5e0e2de241967bc5ee8f855d6072351042688621081d66b2a76b"
|
||||
checksum = "fe413319145d1063f080f27556fd30b1d70b01e2ba10c2a6e40d4be982ffc5d1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"rustc_version 0.4.1",
|
||||
@ -2411,12 +2411,31 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
@ -3127,9 +3146,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "0.25.4"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0d7a6955c7511f60f3ba9e86c6d02b3c3f144f8c24b288d1f4e18074ab8bbec"
|
||||
checksum = "a7e86f5670bd8b028edfb240f0616cad620705b31ec389d55e4f3da2c38dcd48"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@ -3138,6 +3157,7 @@ dependencies = [
|
||||
"futures 0.3.30",
|
||||
"futures-util",
|
||||
"itoa 1.0.11",
|
||||
"num-bigint",
|
||||
"percent-encoding 2.3.1",
|
||||
"pin-project-lite",
|
||||
"ryu",
|
||||
@ -3164,9 +3184,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.10.6"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
|
||||
checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8"
|
||||
dependencies = [
|
||||
"aho-corasick 1.1.3",
|
||||
"memchr",
|
||||
@ -3176,9 +3196,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.7"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
|
||||
checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
|
||||
dependencies = [
|
||||
"aho-corasick 1.1.3",
|
||||
"memchr",
|
||||
@ -3193,9 +3213,9 @@ checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a"
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.4"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
|
||||
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
@ -3335,9 +3355,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.36"
|
||||
version = "0.38.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36"
|
||||
checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
|
||||
dependencies = [
|
||||
"bitflags 2.6.0",
|
||||
"errno",
|
||||
@ -3831,9 +3851,9 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.12.0"
|
||||
version = "3.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
|
||||
checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"fastrand",
|
||||
|
||||
10
Cargo.toml
10
Cargo.toml
@ -47,22 +47,22 @@ mlua = { version = "0.9.9", features = [
|
||||
"luajit",
|
||||
"vendored",
|
||||
], default-features = false }
|
||||
redis = { version = "0.25.4", features = [
|
||||
redis = { version = "0.27.2", features = [
|
||||
"tokio-comp",
|
||||
"connection-manager",
|
||||
"tcp_nodelay"
|
||||
], default-features = false, optional = true }
|
||||
blake3 = { version = "1.5.4", default-features = false }
|
||||
error-stack = { version = "0.4.0", default-features = false, features = [
|
||||
error-stack = { version = "0.5.0", default-features = false, features = [
|
||||
"std",
|
||||
] }
|
||||
async-trait = { version = "0.1.80", default-features = false }
|
||||
regex = { version = "1.9.4", features = ["perf"], default-features = false }
|
||||
regex = { version = "1.11.0", features = ["perf"], default-features = false }
|
||||
futures = { version = "0.3.30", default-features = false, features = ["alloc"] }
|
||||
dhat = { version = "0.3.2", optional = true, default-features = false }
|
||||
mimalloc = { version = "0.1.43", default-features = false }
|
||||
async-once-cell = { version = "0.5.3", default-features = false }
|
||||
actix-governor = { version = "0.5.0", default-features = false }
|
||||
actix-governor = { version = "0.6.0", default-features = false }
|
||||
moka = { version = "0.12.8", optional = true, default-features = false, features = [
|
||||
"future",
|
||||
] }
|
||||
@ -92,7 +92,7 @@ itertools = {version = "0.13.0", default-features = false}
|
||||
[dev-dependencies]
|
||||
rusty-hook = { version = "^0.11.2", default-features = false }
|
||||
criterion = { version = "0.5.1", default-features = false }
|
||||
tempfile = { version = "3.10.1", default-features = false }
|
||||
tempfile = { version = "3.13.0", default-features = false }
|
||||
|
||||
[build-dependencies]
|
||||
lightningcss = { version = "1.0.0-alpha.57", default-features = false, features = [
|
||||
|
||||
@ -48,6 +48,8 @@ pub struct Config {
|
||||
pub tcp_connection_keep_alive: u8,
|
||||
/// It stores the pool idle connection timeout in seconds.
|
||||
pub pool_idle_connection_timeout: u8,
|
||||
/// It stores the number of https connections to keep in the pool.
|
||||
pub number_of_https_connections: u8,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
@ -139,6 +141,7 @@ impl Config {
|
||||
request_timeout: globals.get::<_, u8>("request_timeout")?,
|
||||
tcp_connection_keep_alive: globals.get::<_, u8>("tcp_connection_keep_alive")?,
|
||||
pool_idle_connection_timeout: globals.get::<_, u8>("pool_idle_connection_timeout")?,
|
||||
number_of_https_connections: globals.get::<_, u8>("number_of_https_connections")?,
|
||||
threads,
|
||||
client_connection_keep_alive: globals.get::<_, u8>("client_connection_keep_alive")?,
|
||||
rate_limiter: RateLimiter {
|
||||
|
||||
@ -11,7 +11,9 @@ use thesaurus::synonyms;
|
||||
/// A named struct to store the raw scraped search results scraped search results from the
|
||||
/// upstream search engines before aggregating it.It derives the Clone trait which is needed
|
||||
/// to write idiomatic rust using `Iterators`.
|
||||
///
|
||||
/// (href url in html in simple words).
|
||||
///
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SearchResult {
|
||||
|
||||
@ -81,6 +81,7 @@ pub async fn aggregate(
|
||||
config.pool_idle_connection_timeout as u64,
|
||||
))
|
||||
.tcp_keepalive(Duration::from_secs(config.tcp_connection_keep_alive as u64))
|
||||
.pool_max_idle_per_host(config.number_of_https_connections as usize)
|
||||
.connect_timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
|
||||
.https_only(true)
|
||||
.gzip(true)
|
||||
|
||||
@ -18,6 +18,8 @@ rate_limiter = {
|
||||
}
|
||||
-- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles
|
||||
https_adaptive_window_size = false
|
||||
|
||||
number_of_https_connections = 10 -- the number of https connections that should be available in the connection pool.
|
||||
-- Set keep-alive timer in seconds; keeps clients connected to the HTTP server, different from the connection to upstream search engines
|
||||
client_connection_keep_alive = 120
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user