From 991f3f59de996d99b5bafbebade8f678dee253d9 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 11 Mar 2024 12:01:30 +0300 Subject: [PATCH] :zap: perf: several optimizations for improving the performance of the engine (#540) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * :recycle: refactor: initialize & store the config & cache structs as a constant (#486) - initializes & stores the config & cache structs as a static constant. - Pass the config & cache structs as a static reference to all the functions handling their respective route. * :zap: perf: replace hashmaps with vectors for fetching & aggregating results (#486) - replace hashmaps with vectors for fetching, collecting & aggregating results as it tends to be contigous & cache efficient data structure. - refactor & redesign algorithms for fetching & aggregating results centered around vectors in aggregate function. * :heavy_plus_sign: build: add the future crate (#486) * :zap: perf: use `futureunordered` for collecting results fetched from the tokio spawn tasks (#486) - using the `futureunordered` instead of vector for collecting results reduces the time it takes to fetch the results as the results do not need to come in specific order so any result that gets fetched first gets collected in the `futureunordered` type. Co-authored-by: Spencerjibz * :zap: perf: initialize new async connections parallely using tokio spawn tasks (#486) * :zap: perf: initialize redis pipeline struct once with the default size of 3 (#486) * :zap: perf: reduce branch predictions by reducing conditional code branches (#486) * :white_check_mark: test(unit): provide unit test for the `get_safesearch_level` function (#486) * :zap: perf: reduce clones & use index based loop to improve search results filtering performance (#486) * 🚨 fix(clippy): make clippy/format checks happy (#486) * 🚨 fix(build): make the cargo build check happy (#486) * :zap: perf: reduce the amount of clones, to_owneds & to_strings (#486) * :zap: perf: use async crates & methods & make functions async (#486) * :bookmark: chore(release): bump the app version (#486) --------- Co-authored-by: Spencerjibz --- Cargo.lock | 617 +++++++++--------- Cargo.toml | 12 +- src/bin/websurfx.rs | 13 +- src/cache/cacher.rs | 68 +- src/cache/redis_cacher.rs | 37 +- src/config/parser.rs | 1 - src/engines/bing.rs | 2 +- src/engines/brave.rs | 2 +- src/engines/duckduckgo.rs | 2 +- src/engines/librex.rs | 2 +- src/engines/mojeek.rs | 21 +- src/engines/search_result_parser.rs | 3 +- src/engines/searx.rs | 21 +- src/engines/startpage.rs | 2 +- src/lib.rs | 29 +- src/models/aggregation_models.rs | 4 +- src/models/engine_models.rs | 4 +- src/models/parser_models.rs | 4 +- src/results/aggregator.rs | 191 +++--- src/server/router.rs | 16 +- src/server/routes/search.rs | 144 +++- .../partials/settings_tabs/engines.rs | 4 +- .../partials/settings_tabs/user_interface.rs | 8 +- src/templates/views/search.rs | 2 +- tests/index.rs | 9 +- 25 files changed, 685 insertions(+), 533 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a8f390..322bb34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,17 +4,17 @@ version = 3 [[package]] name = "actix-codec" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" +checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", "bytes 1.5.0", "futures-core", "futures-sink", "memchr", "pin-project-lite", - "tokio 1.35.1", + "tokio 1.36.0", "tokio-util", "tracing", ] @@ -44,7 +44,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web", - "bitflags 2.4.1", + "bitflags 2.4.2", "bytes 1.5.0", "derive_more", "futures-core", @@ -71,24 +71,24 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.5.1" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "129d4c88e98860e1758c5de288d1632b07970a16d59bdf7b8d66053d582bb71f" +checksum = "d223b13fd481fc0d1f83bb12659ae774d9e3601814c68a0bc539731698cca743" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-utils", - "ahash 0.8.7", - "base64 0.21.5", - "bitflags 2.4.1", + "ahash", + "base64 0.21.7", + "bitflags 2.4.2", "brotli", "bytes 1.5.0", "bytestring", "derive_more", "encoding_rs", "futures-core", - "http 0.2.11", + "http 0.2.12", "httparse", "httpdate", "itoa 1.0.10", @@ -100,7 +100,7 @@ dependencies = [ "rand 0.8.5", "sha1", "smallvec 1.13.1", - "tokio 1.35.1", + "tokio 1.36.0", "tokio-util", "tracing", ] @@ -112,7 +112,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -122,7 +122,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d22475596539443685426b6bdadb926ad0ecaefdfc5fb05e5e3441f15463c511" dependencies = [ "bytestring", - "http 0.2.11", + "http 0.2.12", "regex", "serde", "tracing", @@ -135,7 +135,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d" dependencies = [ "futures-core", - "tokio 1.35.1", + "tokio 1.36.0", ] [[package]] @@ -149,9 +149,9 @@ dependencies = [ "actix-utils", "futures-core", "futures-util", - "mio 0.8.10", + "mio 0.8.11", "socket2", - "tokio 1.35.1", + "tokio 1.36.0", "tracing", ] @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.4.1" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e43428f3bf11dee6d166b00ec2df4e3aa8cc1606aaa0b7433c146852e2f4e03b" +checksum = "43a6556ddebb638c2358714d853257ed226ece6023ef9364f23f0c70737ea984" dependencies = [ "actix-codec", "actix-http", @@ -191,7 +191,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web-codegen", - "ahash 0.8.7", + "ahash", "bytes 1.5.0", "bytestring", "cfg-if 1.0.0", @@ -212,7 +212,7 @@ dependencies = [ "serde_urlencoded 0.7.1", "smallvec 1.13.1", "socket2", - "time 0.3.31", + "time 0.3.34", "url 2.5.0", ] @@ -225,7 +225,7 @@ dependencies = [ "actix-router", "proc-macro2 1.0.78", "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -255,20 +255,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.7" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] - -[[package]] -name = "ahash" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if 1.0.0", "getrandom", @@ -318,21 +307,21 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anyhow" -version = "1.0.77" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d19de80eff169429ac1e9f48fffb163916b448a44e8e046186232046d9e1f9" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "7b3d0060af21e8d11a926981cc00c6c1541aa91dd64b9f881985c3da1094425f" [[package]] name = "arrayref" @@ -348,16 +337,16 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "async-compression" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ "brotli", "flate2", "futures-core", "memchr", "pin-project-lite", - "tokio 1.35.1", + "tokio 1.36.0", ] [[package]] @@ -368,13 +357,13 @@ checksum = "9338790e78aa95a416786ec8389546c4b6a1dfc3dc36071ed9518a9413a542eb" [[package]] name = "async-trait" -version = "0.1.76" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2 1.0.78", "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -418,9 +407,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bit-set" @@ -445,9 +434,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "blake3" @@ -494,9 +483,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ "memchr", "serde", @@ -504,9 +493,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" [[package]] name = "bytecount" @@ -557,9 +546,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" +checksum = "694c8807f2ae16faecc43dc17d74b3eb042482789fd0eb64b39a2e04e087053f" dependencies = [ "serde", ] @@ -572,7 +561,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.20", + "semver 1.0.22", "serde", "serde_json", ] @@ -585,12 +574,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.83" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" [[package]] name = "cfg-if" @@ -639,9 +625,9 @@ dependencies = [ [[package]] name = "ciborium" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" dependencies = [ "ciborium-io", "ciborium-ll", @@ -650,15 +636,15 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" [[package]] name = "ciborium-ll" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", "half", @@ -677,18 +663,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.12" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" +checksum = "b230ab84b0ffdf890d5a10abdbc8b83ae1c4918275daea1ab8801f71536b2651" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.4.12" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" dependencies = [ "anstyle", "clap_lex", @@ -696,9 +682,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "cloudabi" @@ -719,7 +705,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio 1.35.1", + "tokio 1.36.0", "tokio-util", ] @@ -772,7 +758,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding 2.3.1", - "time 0.3.31", + "time 0.3.34", "version_check", ] @@ -812,18 +798,18 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if 1.0.0", ] @@ -864,12 +850,11 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.10" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils 0.8.18", + "crossbeam-utils 0.8.19", ] [[package]] @@ -922,12 +907,15 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "crypto-common" @@ -982,7 +970,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1006,9 +994,9 @@ checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", ] @@ -1081,9 +1069,9 @@ checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "encoding_rs" @@ -1105,9 +1093,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.1" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e7cf40684ae96ade6232ed84582f40ce0a66efcd43a5117aef610534f8e0b8" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" dependencies = [ "env_filter", "log", @@ -1347,7 +1335,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2 1.0.78", "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1364,9 +1352,9 @@ checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" @@ -1416,9 +1404,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1439,9 +1427,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "governor" -version = "0.6.0" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "821239e5672ff23e2a7060901fa622950bbd80b649cdaadd78d1c1767ed14eb4" +checksum = "68a7f542ee6b35af73b06abc0dad1c1bae89964e4e253bc4b587b91c9637867b" dependencies = [ "cfg-if 1.0.0", "dashmap", @@ -1450,9 +1438,11 @@ dependencies = [ "no-std-compat", "nonzero_ext", "parking_lot 0.12.1", + "portable-atomic", "quanta", "rand 0.8.5", "smallvec 1.13.1", + "spinning_top", ] [[package]] @@ -1475,28 +1465,32 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes 1.5.0", "fnv", "futures-core", "futures-sink", "futures-util", - "http 0.2.11", - "indexmap 2.1.0", + "http 0.2.12", + "indexmap 2.2.5", "slab", - "tokio 1.35.1", + "tokio 1.36.0", "tokio-util", "tracing", ] [[package]] name = "half" -version = "1.8.2" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" +dependencies = [ + "cfg-if 1.0.0", + "crunchy", +] [[package]] name = "hashbrown" @@ -1510,7 +1504,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.7", + "ahash", "bumpalo", ] @@ -1522,9 +1516,9 @@ checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "home" @@ -1576,9 +1570,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes 1.5.0", "fnv", @@ -1604,7 +1598,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes 1.5.0", - "http 0.2.11", + "http 0.2.12", "pin-project-lite", ] @@ -1666,15 +1660,15 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.22", - "http 0.2.11", + "h2 0.3.24", + "http 0.2.12", "http-body 0.4.6", "httparse", "httpdate", "itoa 1.0.10", "pin-project-lite", "socket2", - "tokio 1.35.1", + "tokio 1.36.0", "tower-service", "tracing", "want 0.3.1", @@ -1687,10 +1681,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http 0.2.11", + "http 0.2.12", "hyper 0.14.28", "rustls", - "tokio 1.35.1", + "tokio 1.36.0", "tokio-rustls", ] @@ -1751,9 +1745,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -1785,12 +1779,12 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ "hermit-abi", - "rustix", + "libc", "windows-sys 0.52.0", ] @@ -1817,9 +1811,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -1864,16 +1858,17 @@ dependencies = [ [[package]] name = "lightningcss" -version = "1.0.0-alpha.52" +version = "1.0.0-alpha.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771a62dedf5ec563bbfea9760f6c6a6bc546e67355eba0cd7d00c0dc34b11d90" +checksum = "07d306844e5af1753490c420c0d6ae3d814b00725092d106332762827ca8f0fe" dependencies = [ - "ahash 0.7.7", - "bitflags 2.4.1", + "ahash", + "bitflags 2.4.2", "const-str", "cssparser 0.33.0", "cssparser-color", "data-encoding", + "getrandom", "itertools", "lazy_static", "parcel_selectors", @@ -1884,9 +1879,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "local-channel" @@ -1941,9 +1936,9 @@ dependencies = [ [[package]] name = "luajit-src" -version = "210.5.3+29b0b28" +version = "210.5.6+9cc2e42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2bb89013916ce5c949f01a1fbd6d435a58e1d980767a791d755911211d792d" +checksum = "23b365d859c9ffc187f48bb3e25ec80c3b40cf3f68f53544f4adeaee70554157" dependencies = [ "cc", "which", @@ -1955,15 +1950,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" -[[package]] -name = "mach2" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" -dependencies = [ - "libc", -] - [[package]] name = "markup5ever" version = "0.8.1" @@ -2073,12 +2059,12 @@ dependencies = [ [[package]] name = "mini-moka" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e0b72e7c9042467008b10279fc732326bd605459ae03bda88825909dd19b56" +checksum = "c325dfab65f261f386debee8b0969da215b3fa0037e74c8a1234db7ba986d803" dependencies = [ "crossbeam-channel", - "crossbeam-utils 0.8.18", + "crossbeam-utils 0.8.19", "dashmap", "skeptic", "smallvec 1.13.1", @@ -2099,22 +2085,18 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] [[package]] name = "mintex" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7c5ba1c3b5a23418d7bbf98c71c3d4946a0125002129231da8d6b723d559cb" -dependencies = [ - "once_cell", - "sys-info", -] +checksum = "9bec4598fddb13cc7b528819e697852653252b760f1228b7642679bf2ff2cd07" [[package]] name = "mio" @@ -2137,9 +2119,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", @@ -2161,9 +2143,9 @@ dependencies = [ [[package]] name = "mlua" -version = "0.9.2" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c81f8ac20188feb5461a73eabb22a34dd09d6d58513535eb587e46bff6ba250" +checksum = "868d02cb5eb97761bbf6bd6922c1c7a88b8ea252bbf43bd8350a0bf8497a1fc0" dependencies = [ "bstr", "mlua-sys", @@ -2174,9 +2156,9 @@ dependencies = [ [[package]] name = "mlua-sys" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc29228347d6bdc9e613dc95c69df2817f755434ee0f7f3b27b57755fe238b7f" +checksum = "2847b42764435201d8cbee1f517edb79c4cca4181877b90047587c89e1b7bce4" dependencies = [ "cc", "cfg-if 1.0.0", @@ -2239,10 +2221,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" [[package]] -name = "num-traits" -version = "0.2.17" +name = "num-conv" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg 1.1.0", ] @@ -2280,17 +2268,17 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "opaque-debug" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.62" +version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if 1.0.0", "foreign-types", "libc", @@ -2307,7 +2295,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2 1.0.78", "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2318,9 +2306,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.98" +version = "0.9.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" dependencies = [ "cc", "libc", @@ -2334,7 +2322,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05d74befe2d076330d9a58bf9ca2da424568724ab278adf15fb5718253133887" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cssparser 0.33.0", "fxhash", "log", @@ -2518,7 +2506,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2 1.0.78", "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2550,22 +2538,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2 1.0.78", "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2582,9 +2570,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "poly1305" @@ -2597,6 +2585,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "powerfmt" version = "0.2.0" @@ -2669,24 +2663,23 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.9.3" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" +checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", "memchr", "unicase", ] [[package]] name = "quanta" -version = "0.11.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +checksum = "9ca0b7bac0b97248c40bb77288fc52029cf1459c0461ea1b05ee32ccf011de2c" dependencies = [ - "crossbeam-utils 0.8.18", + "crossbeam-utils 0.8.19", "libc", - "mach2", "once_cell", "raw-cpuid", "wasi 0.11.0+wasi-snapshot-preview1", @@ -2850,11 +2843,11 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "10.7.0" +version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", ] [[package]] @@ -2882,7 +2875,7 @@ dependencies = [ "percent-encoding 2.3.1", "pin-project-lite", "ryu", - "tokio 1.35.1", + "tokio 1.36.0", "tokio-retry", "tokio-util", "url 2.5.0", @@ -2905,9 +2898,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick 1.1.2", "memchr", @@ -2917,9 +2910,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick 1.1.2", "memchr", @@ -2968,18 +2961,18 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.24" +version = "0.11.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +checksum = "0eea5a9eb898d3783f17c6407670e3592fd174cb81a10e51d4c37f49450b9946" dependencies = [ "async-compression", - "base64 0.21.5", + "base64 0.21.7", "bytes 1.5.0", "encoding_rs", "futures-core", "futures-util", - "h2 0.3.22", - "http 0.2.11", + "h2 0.3.24", + "http 0.2.12", "http-body 0.4.6", "hyper 0.14.28", "hyper-rustls", @@ -2997,7 +2990,7 @@ dependencies = [ "serde_urlencoded 0.7.1", "sync_wrapper", "system-configuration", - "tokio 1.35.1", + "tokio 1.36.0", "tokio-rustls", "tokio-util", "tower-service", @@ -3011,16 +3004,17 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if 1.0.0", "getrandom", "libc", "spin", "untrusted", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -3050,7 +3044,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.20", + "semver 1.0.22", ] [[package]] @@ -3059,7 +3053,7 @@ version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -3084,7 +3078,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", ] [[package]] @@ -3111,9 +3105,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "same-file" @@ -3145,7 +3139,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "585480e3719b311b78a573db1c9d9c4c1f8010c2dee4cc59c2efe58ea4dbc3e1" dependencies = [ - "ahash 0.8.7", + "ahash", "cssparser 0.31.2", "ego-tree", "html5ever 0.26.0", @@ -3203,7 +3197,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cssparser 0.31.2", "derive_more", "fxhash", @@ -3227,9 +3221,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" dependencies = [ "serde", ] @@ -3242,29 +3236,29 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.196" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.196" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2 1.0.78", "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] name = "serde_json" -version = "1.0.109" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0652c533506ad7a2e353cce269330d6afd8bdfb6d75e0ace5b35aacbd7b9e9" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ "itoa 1.0.10", "ryu", @@ -3380,12 +3374,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -3394,6 +3388,15 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api 0.4.11", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -3499,9 +3502,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" dependencies = [ "proc-macro2 1.0.78", "quote 1.0.35", @@ -3526,32 +3529,22 @@ dependencies = [ "unicode-xid 0.2.4", ] -[[package]] -name = "sys-info" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "system-configuration" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", "core-foundation", "system-configuration-sys", ] [[package]] name = "system-configuration-sys" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ "core-foundation-sys", "libc", @@ -3605,12 +3598,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "itoa 1.0.10", + "num-conv", "powerfmt", "serde", "time-core", @@ -3625,10 +3619,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] @@ -3678,14 +3673,14 @@ dependencies = [ [[package]] name = "tokio" -version = "1.35.1" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes 1.5.0", "libc", - "mio 0.8.10", + "mio 0.8.11", "num_cpus", "parking_lot 0.12.1", "pin-project-lite", @@ -3745,7 +3740,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2 1.0.78", "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -3775,7 +3770,7 @@ checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" dependencies = [ "pin-project", "rand 0.8.5", - "tokio 1.35.1", + "tokio 1.36.0", ] [[package]] @@ -3785,7 +3780,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", - "tokio 1.35.1", + "tokio 1.36.0", ] [[package]] @@ -3851,7 +3846,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.35.1", + "tokio 1.36.0", "tracing", ] @@ -3928,9 +3923,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -3940,9 +3935,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] @@ -4038,9 +4033,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -4080,9 +4075,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -4090,24 +4085,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2 1.0.78", "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -4117,9 +4112,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote 1.0.35", "wasm-bindgen-macro-support", @@ -4127,28 +4122,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2 1.0.78", "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -4156,23 +4151,23 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.3" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "websurfx" -version = "1.9.20" +version = "1.10.9" dependencies = [ "actix-cors", "actix-files", "actix-governor", "actix-web", + "async-compression", "async-once-cell", "async-trait", - "base64 0.21.5", + "base64 0.21.7", "blake3", - "brotli", "cfg-if 1.0.0", "chacha20", "chacha20poly1305", @@ -4191,27 +4186,27 @@ dependencies = [ "mlua", "redis", "regex", - "reqwest 0.11.24", + "reqwest 0.11.25", "rusty-hook", "scraper", "serde", "serde_json", "smallvec 1.13.1", "tempfile", - "tokio 1.35.1", + "tokio 1.36.0", ] [[package]] name = "which" -version = "5.0.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bf3ea8596f3a0dd5980b46430f2058dfe2c36a27ccfbb1845d6fbfcd9ba6e14" +checksum = "7fa5e0c10bf77f44aac573e498d1a82d5fbd5e91f6fc0a99e7be4b38e85e101c" dependencies = [ "either", "home", "once_cell", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4272,7 +4267,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.4", ] [[package]] @@ -4292,17 +4287,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -4313,9 +4308,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" @@ -4325,9 +4320,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" @@ -4337,9 +4332,9 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" @@ -4349,9 +4344,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" @@ -4361,9 +4356,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" [[package]] name = "windows_x86_64_gnullvm" @@ -4373,9 +4368,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" @@ -4385,9 +4380,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "winreg" @@ -4435,7 +4430,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2 1.0.78", "quote 1.0.35", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 76d0059..556c5d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "1.9.20" +version = "1.10.9" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" @@ -14,7 +14,7 @@ path = "src/bin/websurfx.rs" [dependencies] reqwest = {version="0.11.24", default-features=false, features=["rustls-tls","brotli", "gzip"]} -tokio = {version="1.32.0",features=["rt-multi-thread","macros"], default-features = false} +tokio = {version="1.32.0",features=["rt-multi-thread","macros", "fs", "io-util"], default-features = false} serde = {version="1.0.196", default-features=false, features=["derive"]} serde_json = {version="1.0.109", default-features=false} maud = {version="0.25.0", default-features=false, features=["actix-web"]} @@ -32,13 +32,13 @@ error-stack = {version="0.4.0", default-features=false, features=["std"]} async-trait = {version="0.1.76", default-features=false} regex = {version="1.9.4", features=["perf"], default-features = false} smallvec = {version="1.13.1", features=["union", "serde"], default-features=false} -futures = {version="0.3.28", default-features=false} -dhat = {version="0.3.3", optional = true, default-features=false} +futures = {version="0.3.30", default-features=false, features=["alloc"]} +dhat = {version="0.3.2", optional = true, default-features=false} mimalloc = { version = "0.1.38", default-features = false } async-once-cell = {version="0.5.3", default-features=false} actix-governor = {version="0.5.0", default-features=false} mini-moka = { version="0.10", optional = true, default-features=false, features=["sync"]} -brotli = { version = "3.4.0", default-features = false, features=["std"], optional=true} +async-compression = { version = "0.4.6", default-features = false, features=["brotli","tokio"], optional=true} chacha20poly1305={version="0.10.1", default-features=false, features=["alloc","getrandom"], optional=true} chacha20 = {version="0.9.1", default-features=false, optional=true} base64 = {version="0.21.5", default-features=false, features=["std"], optional=true} @@ -84,7 +84,7 @@ default = ["memory-cache"] dhat-heap = ["dep:dhat"] memory-cache = ["dep:mini-moka"] redis-cache = ["dep:redis","dep:base64"] -compress-cache-results = ["dep:brotli","dep:cfg-if"] +compress-cache-results = ["dep:async-compression","dep:cfg-if"] encrypt-cache-results = ["dep:chacha20poly1305","dep:chacha20"] cec-cache-results = ["compress-cache-results","encrypt-cache-results"] diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index 1852695..5abf2cc 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -5,7 +5,7 @@ #[cfg(not(feature = "dhat-heap"))] use mimalloc::MiMalloc; -use std::net::TcpListener; +use std::{net::TcpListener, sync::OnceLock}; use websurfx::{cache::cacher::create_cache, config::parser::Config, run}; /// A dhat heap memory profiler @@ -17,6 +17,9 @@ static ALLOC: dhat::Alloc = dhat::Alloc; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; +/// A static constant for holding the parsed config. +static CONFIG: OnceLock = OnceLock::new(); + /// The function that launches the main server and registers all the routes of the website. /// /// # Error @@ -29,10 +32,10 @@ async fn main() -> std::io::Result<()> { #[cfg(feature = "dhat-heap")] let _profiler = dhat::Profiler::new_heap(); - // Initialize the parsed config file. - let config = Config::parse(false).unwrap(); + // Initialize the parsed config globally. + let config = CONFIG.get_or_init(|| Config::parse(false).unwrap()); - let cache = create_cache(&config).await; + let cache = create_cache(config).await; log::info!( "started server on port {} and IP {}", @@ -45,7 +48,7 @@ async fn main() -> std::io::Result<()> { config.port, ); - let listener = TcpListener::bind((config.binding_ip.clone(), config.port))?; + let listener = TcpListener::bind((config.binding_ip.as_str(), config.port))?; run(listener, config, cache)?.await } diff --git a/src/cache/cacher.rs b/src/cache/cacher.rs index 53d8050..f027aff 100644 --- a/src/cache/cacher.rs +++ b/src/cache/cacher.rs @@ -93,7 +93,7 @@ pub trait Cacher: Send + Sync { feature = "encrypt-cache-results", feature = "cec-cache-results" ))] - fn encrypt_or_decrypt_results( + async fn encrypt_or_decrypt_results( &mut self, mut bytes: Vec, encrypt: bool, @@ -137,11 +137,19 @@ pub trait Cacher: Send + Sync { /// Returns the compressed bytes on success otherwise it returns a CacheError /// on failure. #[cfg(any(feature = "compress-cache-results", feature = "cec-cache-results"))] - fn compress_results(&mut self, mut bytes: Vec) -> Result, Report> { - use std::io::Write; - let mut writer = brotli::CompressorWriter::new(Vec::new(), 4096, 11, 22); + async fn compress_results( + &mut self, + mut bytes: Vec, + ) -> Result, Report> { + use tokio::io::AsyncWriteExt; + let mut writer = async_compression::tokio::write::BrotliEncoder::new(Vec::new()); writer .write_all(&bytes) + .await + .map_err(|_| CacheError::CompressionError)?; + writer + .shutdown() + .await .map_err(|_| CacheError::CompressionError)?; bytes = writer.into_inner(); Ok(bytes) @@ -159,17 +167,17 @@ pub trait Cacher: Send + Sync { /// Returns the compressed and encrypted bytes on success otherwise it returns a CacheError /// on failure. #[cfg(feature = "cec-cache-results")] - fn compress_encrypt_compress_results( + async fn compress_encrypt_compress_results( &mut self, mut bytes: Vec, ) -> Result, Report> { // compress first - bytes = self.compress_results(bytes)?; + bytes = self.compress_results(bytes).await?; // encrypt - bytes = self.encrypt_or_decrypt_results(bytes, true)?; + bytes = self.encrypt_or_decrypt_results(bytes, true).await?; // compress again; - bytes = self.compress_results(bytes)?; + bytes = self.compress_results(bytes).await?; Ok(bytes) } @@ -187,11 +195,11 @@ pub trait Cacher: Send + Sync { /// on failure. #[cfg(any(feature = "compress-cache-results", feature = "cec-cache-results"))] - fn decompress_results(&mut self, bytes: &[u8]) -> Result, Report> { + async fn decompress_results(&mut self, bytes: &[u8]) -> Result, Report> { cfg_if::cfg_if! { if #[cfg(feature = "compress-cache-results")] { - decompress_util(bytes) + decompress_util(bytes).await } else if #[cfg(feature = "cec-cache-results")] @@ -199,7 +207,7 @@ pub trait Cacher: Send + Sync { let decompressed = decompress_util(bytes)?; let decrypted = self.encrypt_or_decrypt_results(decompressed, false)?; - decompress_util(&decrypted) + decompress_util(&decrypted).await } } @@ -216,7 +224,7 @@ pub trait Cacher: Send + Sync { /// # Error /// Returns a Vec of compressed or encrypted bytes on success otherwise it returns a CacheError /// on failure. - fn pre_process_search_results( + async fn pre_process_search_results( &mut self, search_results: &SearchResults, ) -> Result, Report> { @@ -224,19 +232,20 @@ pub trait Cacher: Send + Sync { let mut bytes: Vec = search_results.try_into()?; #[cfg(feature = "compress-cache-results")] { - let compressed = self.compress_results(bytes)?; + let compressed = self.compress_results(bytes).await?; bytes = compressed; } #[cfg(feature = "encrypt-cache-results")] { - let encrypted = self.encrypt_or_decrypt_results(bytes, true)?; + let encrypted = self.encrypt_or_decrypt_results(bytes, true).await?; bytes = encrypted; } #[cfg(feature = "cec-cache-results")] { - let compressed_encrypted_compressed = self.compress_encrypt_compress_results(bytes)?; + let compressed_encrypted_compressed = + self.compress_encrypt_compress_results(bytes).await?; bytes = compressed_encrypted_compressed; } @@ -256,25 +265,25 @@ pub trait Cacher: Send + Sync { /// on failure. #[allow(unused_mut)] // needs to be mutable when any of the features is enabled - fn post_process_search_results( + async fn post_process_search_results( &mut self, mut bytes: Vec, ) -> Result> { #[cfg(feature = "compress-cache-results")] { - let decompressed = self.decompress_results(&bytes)?; + let decompressed = self.decompress_results(&bytes).await?; bytes = decompressed } #[cfg(feature = "encrypt-cache-results")] { - let decrypted = self.encrypt_or_decrypt_results(bytes, false)?; + let decrypted = self.encrypt_or_decrypt_results(bytes, false).await?; bytes = decrypted } #[cfg(feature = "cec-cache-results")] { - let decompressed_decrypted = self.decompress_results(&bytes)?; + let decompressed_decrypted = self.decompress_results(&bytes).await?; bytes = decompressed_decrypted; } @@ -295,16 +304,19 @@ pub trait Cacher: Send + Sync { /// on failure. #[cfg(any(feature = "compress-cache-results", feature = "cec-cache-results"))] -fn decompress_util(input: &[u8]) -> Result, Report> { - use std::io::Write; - let mut writer = brotli::DecompressorWriter::new(Vec::new(), 4096); +async fn decompress_util(input: &[u8]) -> Result, Report> { + use tokio::io::AsyncWriteExt; + let mut writer = async_compression::tokio::write::BrotliDecoder::new(Vec::new()); writer .write_all(input) + .await .map_err(|_| CacheError::CompressionError)?; - let bytes = writer - .into_inner() + writer + .shutdown() + .await .map_err(|_| CacheError::CompressionError)?; + let bytes = writer.into_inner(); Ok(bytes) } @@ -329,7 +341,7 @@ impl Cacher for RedisCache { let bytes = base64::engine::general_purpose::STANDARD_NO_PAD .decode(base64_string) .map_err(|_| CacheError::Base64DecodingOrEncodingError)?; - self.post_process_search_results(bytes) + self.post_process_search_results(bytes).await } async fn cache_results( @@ -345,7 +357,7 @@ impl Cacher for RedisCache { let mut bytes = Vec::with_capacity(search_results_len); for result in search_results { - let processed = self.pre_process_search_results(result)?; + let processed = self.pre_process_search_results(result).await?; bytes.push(processed); } @@ -405,7 +417,7 @@ impl Cacher for InMemoryCache { async fn cached_results(&mut self, url: &str) -> Result> { let hashed_url_string = self.hash_url(url); match self.cache.get(&hashed_url_string) { - Some(res) => self.post_process_search_results(res), + Some(res) => self.post_process_search_results(res).await, None => Err(Report::new(CacheError::MissingValue)), } } @@ -417,7 +429,7 @@ impl Cacher for InMemoryCache { ) -> Result<(), Report> { for (url, search_result) in urls.iter().zip(search_results.iter()) { let hashed_url_string = self.hash_url(url); - let bytes = self.pre_process_search_results(search_result)?; + let bytes = self.pre_process_search_results(search_result).await?; self.cache.insert(hashed_url_string, bytes); } diff --git a/src/cache/redis_cacher.rs b/src/cache/redis_cacher.rs index c775963..1dd7cb8 100644 --- a/src/cache/redis_cacher.rs +++ b/src/cache/redis_cacher.rs @@ -1,15 +1,16 @@ //! This module provides the functionality to cache the aggregated results fetched and aggregated //! from the upstream search engines in a json format. +use super::error::CacheError; use error_stack::Report; -use futures::future::try_join_all; +use futures::stream::FuturesUnordered; use redis::{aio::ConnectionManager, AsyncCommands, Client, RedisError}; -use super::error::CacheError; +/// A constant holding the redis pipeline size. +const REDIS_PIPELINE_SIZE: usize = 3; /// A named struct which stores the redis Connection url address to which the client will /// connect to. -#[derive(Clone)] pub struct RedisCache { /// It stores a pool of connections ready to be used. connection_pool: Vec, @@ -20,6 +21,8 @@ pub struct RedisCache { current_connection: u8, /// It stores the max TTL for keys. cache_ttl: u16, + /// It stores the redis pipeline struct of size 3. + pipeline: redis::Pipeline, } impl RedisCache { @@ -30,6 +33,8 @@ impl RedisCache { /// * `redis_connection_url` - It takes the redis Connection url address. /// * `pool_size` - It takes the size of the connection pool (in other words the number of /// connections that should be stored in the pool). + /// * `cache_ttl` - It takes the the time to live for cached results to live in the redis + /// server. /// /// # Error /// @@ -41,18 +46,28 @@ impl RedisCache { cache_ttl: u16, ) -> Result> { let client = Client::open(redis_connection_url)?; - let mut tasks: Vec<_> = Vec::new(); + let tasks: FuturesUnordered<_> = FuturesUnordered::new(); for _ in 0..pool_size { - tasks.push(client.get_connection_manager()); + let client_partially_cloned = client.clone(); + tasks.push(tokio::spawn(async move { + client_partially_cloned.get_connection_manager().await + })); + } + + let mut outputs = Vec::new(); + for task in tasks { + outputs.push(task.await??); } let redis_cache = RedisCache { - connection_pool: try_join_all(tasks).await?, + connection_pool: outputs, pool_size, current_connection: Default::default(), cache_ttl, + pipeline: redis::Pipeline::with_capacity(REDIS_PIPELINE_SIZE), }; + Ok(redis_cache) } @@ -122,13 +137,14 @@ impl RedisCache { keys: impl Iterator, ) -> Result<(), Report> { self.current_connection = Default::default(); - let mut pipeline = redis::Pipeline::with_capacity(3); for (key, json_result) in keys.zip(json_results) { - pipeline.set_ex(key, json_result, self.cache_ttl.into()); + self.pipeline + .set_ex(key, json_result, self.cache_ttl.into()); } - let mut result: Result<(), RedisError> = pipeline + let mut result: Result<(), RedisError> = self + .pipeline .query_async(&mut self.connection_pool[self.current_connection as usize]) .await; @@ -149,7 +165,8 @@ impl RedisCache { CacheError::PoolExhaustionWithConnectionDropError, )); } - result = pipeline + result = self + .pipeline .query_async( &mut self.connection_pool[self.current_connection as usize], ) diff --git a/src/config/parser.rs b/src/config/parser.rs index 4494db3..fa61ce0 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -9,7 +9,6 @@ use mlua::Lua; use std::{collections::HashMap, fs, thread::available_parallelism}; /// A named struct which stores the parsed config file options. -#[derive(Clone)] pub struct Config { /// It stores the parsed port number option on which the server should launch. pub port: u16, diff --git a/src/engines/bing.rs b/src/engines/bing.rs index 84dbf93..ec582e4 100644 --- a/src/engines/bing.rs +++ b/src/engines/bing.rs @@ -48,7 +48,7 @@ impl SearchEngine for Bing { user_agent: &str, client: &Client, _safe_search: u8, - ) -> Result, EngineError> { + ) -> Result, EngineError> { // Bing uses `start results from this number` convention // So, for 10 results per page, page 0 starts at 1, page 1 // starts at 11, and so on. diff --git a/src/engines/brave.rs b/src/engines/brave.rs index 49626e3..65067fc 100644 --- a/src/engines/brave.rs +++ b/src/engines/brave.rs @@ -44,7 +44,7 @@ impl SearchEngine for Brave { user_agent: &str, client: &Client, safe_search: u8, - ) -> Result, EngineError> { + ) -> Result, EngineError> { let url = format!("https://search.brave.com/search?q={query}&offset={page}"); let safe_search_level = match safe_search { diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index c48522f..02ee481 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -47,7 +47,7 @@ impl SearchEngine for DuckDuckGo { user_agent: &str, client: &Client, _safe_search: u8, - ) -> Result, EngineError> { + ) -> Result, EngineError> { // Page number can be missing or empty string and so appropriate handling is required // so that upstream server recieves valid page number. let url: String = match page { diff --git a/src/engines/librex.rs b/src/engines/librex.rs index b34393f..69e4611 100644 --- a/src/engines/librex.rs +++ b/src/engines/librex.rs @@ -62,7 +62,7 @@ impl SearchEngine for LibreX { user_agent: &str, client: &Client, _safe_search: u8, - ) -> Result, EngineError> { + ) -> Result, EngineError> { // Page number can be missing or empty string and so appropriate handling is required // so that upstream server recieves valid page number. let url: String = format!( diff --git a/src/engines/mojeek.rs b/src/engines/mojeek.rs index 3f7fbb1..e376828 100644 --- a/src/engines/mojeek.rs +++ b/src/engines/mojeek.rs @@ -47,7 +47,7 @@ impl SearchEngine for Mojeek { user_agent: &str, client: &Client, safe_search: u8, - ) -> Result, EngineError> { + ) -> Result, EngineError> { // Mojeek uses `start results from this number` convention // So, for 10 results per page, page 0 starts at 1, page 1 // starts at 11, and so on. @@ -72,8 +72,23 @@ impl SearchEngine for Mojeek { "Yep", "You", ]; + let qss = search_engines.join("%2C"); - let safe = if safe_search == 0 { "0" } else { "1" }; + + // A branchless condition to check whether the `safe_search` parameter has the + // value 0 or not. If it is zero then it sets the value 0 otherwise it sets + // the value to 1 for all other values of `safe_search` + // + // Moreover, the below branchless code is equivalent to the following code below: + // + // ```rust + // let safe = if safe_search == 0 { 0 } else { 1 }.to_string(); + // ``` + // + // For more information on branchless programming. See: + // + // * https://piped.video/watch?v=bVJ-mWWL7cE + let safe = u8::from(safe_search != 0).to_string(); // Mojeek detects automated requests, these are preferences that are // able to circumvent the countermeasure. Some of these are @@ -89,7 +104,7 @@ impl SearchEngine for Mojeek { ("hp", "minimal"), ("lb", "en"), ("qss", &qss), - ("safe", safe), + ("safe", &safe), ]; let mut query_params_string = String::new(); diff --git a/src/engines/search_result_parser.rs b/src/engines/search_result_parser.rs index 0512bdd..8c16b65 100644 --- a/src/engines/search_result_parser.rs +++ b/src/engines/search_result_parser.rs @@ -1,5 +1,4 @@ //! This modules provides helper functionalities for parsing a html document into internal SearchResult. -use std::collections::HashMap; use crate::models::{aggregation_models::SearchResult, engine_models::EngineError}; use error_stack::{Report, Result}; @@ -47,7 +46,7 @@ impl SearchResultParser { &self, document: &Html, builder: impl Fn(&ElementRef<'_>, &ElementRef<'_>, &ElementRef<'_>) -> Option, - ) -> Result, EngineError> { + ) -> Result, EngineError> { let res = document .select(&self.results) .filter_map(|result| { diff --git a/src/engines/searx.rs b/src/engines/searx.rs index 9bb297c..df96857 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -43,12 +43,21 @@ impl SearchEngine for Searx { user_agent: &str, client: &Client, mut safe_search: u8, - ) -> Result, EngineError> { - // Page number can be missing or empty string and so appropriate handling is required - // so that upstream server recieves valid page number. - if safe_search == 3 { - safe_search = 2; - }; + ) -> Result, EngineError> { + // A branchless condition to check whether the `safe_search` parameter has the + // value greater than equal to three or not. If it is, then it modifies the + // `safesearch` parameters value to 2. + // + // Moreover, the below branchless code is equivalent to the following code below: + // + // ```rust + // safe_search = u8::from(safe_search == 3) * 2; + // ``` + // + // For more information on branchless programming. See: + // + // * https://piped.video/watch?v=bVJ-mWWL7cE + safe_search = u8::from(safe_search >= 3) * 2; let url: String = format!( "https://searx.be/search?q={query}&pageno={}&safesearch={safe_search}", diff --git a/src/engines/startpage.rs b/src/engines/startpage.rs index 540a0ce..97b7a40 100644 --- a/src/engines/startpage.rs +++ b/src/engines/startpage.rs @@ -47,7 +47,7 @@ impl SearchEngine for Startpage { user_agent: &str, client: &Client, _safe_search: u8, - ) -> Result, EngineError> { + ) -> Result, EngineError> { // Page number can be missing or empty string and so appropriate handling is required // so that upstream server recieves valid page number. let url: String = format!( diff --git a/src/lib.rs b/src/lib.rs index ec35273..1642d69 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,7 @@ pub mod results; pub mod server; pub mod templates; -use std::net::TcpListener; +use std::{net::TcpListener, sync::OnceLock}; use crate::server::router; @@ -31,6 +31,9 @@ use cache::cacher::{Cacher, SharedCache}; use config::parser::Config; use handler::{file_path, FileType}; +/// A static constant for holding the cache struct. +static SHARED_CACHE: OnceLock = OnceLock::new(); + /// Runs the web server on the provided TCP listener and returns a `Server` instance. /// /// # Arguments @@ -44,27 +47,29 @@ use handler::{file_path, FileType}; /// # Example /// /// ```rust -/// use std::net::TcpListener; +/// use std::{net::TcpListener, sync::OnceLock}; /// use websurfx::{config::parser::Config, run, cache::cacher::create_cache}; /// +/// /// A static constant for holding the parsed config. +/// static CONFIG: OnceLock = OnceLock::new(); +/// /// #[tokio::main] /// async fn main(){ -/// let config = Config::parse(true).unwrap(); +/// // Initialize the parsed config globally. +/// let config = CONFIG.get_or_init(|| Config::parse(true).unwrap()); /// let listener = TcpListener::bind("127.0.0.1:8080").expect("Failed to bind address"); -/// let cache = create_cache(&config).await; -/// let server = run(listener,config,cache).expect("Failed to start server"); +/// let cache = create_cache(config).await; +/// let server = run(listener,&config,cache).expect("Failed to start server"); /// } /// ``` pub fn run( listener: TcpListener, - config: Config, + config: &'static Config, cache: impl Cacher + 'static, ) -> std::io::Result { let public_folder_path: &str = file_path(FileType::Theme)?; - let cloned_config_threads_opt: u8 = config.threads; - - let cache = web::Data::new(SharedCache::new(cache)); + let cache = SHARED_CACHE.get_or_init(|| SharedCache::new(cache)); let server = HttpServer::new(move || { let cors: Cors = Cors::default() @@ -81,8 +86,8 @@ pub fn run( // Compress the responses provided by the server for the client requests. .wrap(Compress::default()) .wrap(Logger::default()) // added logging middleware for logging. - .app_data(web::Data::new(config.clone())) - .app_data(cache.clone()) + .app_data(web::Data::new(config)) + .app_data(web::Data::new(cache)) .wrap(cors) .wrap(Governor::new( &GovernorConfigBuilder::default() @@ -107,7 +112,7 @@ pub fn run( .service(router::settings) // settings page .default_service(web::route().to(router::not_found)) // error page }) - .workers(cloned_config_threads_opt as usize) + .workers(config.threads as usize) // Start server on 127.0.0.1 with the user provided port number. for example 127.0.0.1:8080. .listen(listener)? .run(); diff --git a/src/models/aggregation_models.rs b/src/models/aggregation_models.rs index 680d222..6be3958 100644 --- a/src/models/aggregation_models.rs +++ b/src/models/aggregation_models.rs @@ -154,8 +154,8 @@ impl SearchResults { } /// A setter function that sets the filtered to true. - pub fn set_filtered(&mut self) { - self.filtered = true; + pub fn set_filtered(&mut self, filtered: bool) { + self.filtered = filtered; } /// A getter function that gets the value of `engine_errors_info`. diff --git a/src/models/engine_models.rs b/src/models/engine_models.rs index 4d56836..932afce 100644 --- a/src/models/engine_models.rs +++ b/src/models/engine_models.rs @@ -4,7 +4,7 @@ use super::aggregation_models::SearchResult; use error_stack::{Report, Result, ResultExt}; use reqwest::Client; -use std::{collections::HashMap, fmt}; +use std::fmt; /// A custom error type used for handle engine associated errors. #[derive(Debug)] @@ -147,7 +147,7 @@ pub trait SearchEngine: Sync + Send { user_agent: &str, client: &Client, safe_search: u8, - ) -> Result, EngineError>; + ) -> Result, EngineError>; } /// A named struct which stores the engine struct with the name of the associated engine. diff --git a/src/models/parser_models.rs b/src/models/parser_models.rs index 75a4c13..24b3fd8 100644 --- a/src/models/parser_models.rs +++ b/src/models/parser_models.rs @@ -10,7 +10,7 @@ /// order to allow the deserializing the json back to struct in aggregate function in /// aggregator.rs and create a new struct out of it and then serialize it back to json and pass /// it to the template files. -#[derive(Clone, Default)] +#[derive(Default)] pub struct Style { /// It stores the parsed theme option used to set a theme for the website. pub theme: String, @@ -40,7 +40,6 @@ impl Style { } /// Configuration options for the aggregator. -#[derive(Clone)] pub struct AggregatorConfig { /// It stores the option to whether enable or disable random delays between /// requests. @@ -48,7 +47,6 @@ pub struct AggregatorConfig { } /// Configuration options for the rate limiter middleware. -#[derive(Clone)] pub struct RateLimiter { /// The number of request that are allowed within a provided time limit. pub number_of_requests: u8, diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index a95fb33..b46befd 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -9,22 +9,25 @@ use crate::models::{ engine_models::{EngineError, EngineHandler}, }; use error_stack::Report; +use futures::stream::FuturesUnordered; use regex::Regex; use reqwest::{Client, ClientBuilder}; +use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -use std::{ - collections::HashMap, - io::{BufReader, Read}, +use tokio::{ + fs::File, + io::{AsyncBufReadExt, BufReader}, + task::JoinHandle, time::Duration, }; -use std::{fs::File, io::BufRead}; -use tokio::task::JoinHandle; /// A constant for holding the prebuilt Client globally in the app. static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); /// Aliases for long type annotations -type FutureVec = Vec, Report>>>; + +type FutureVec = + FuturesUnordered, Report>>>; /// The function aggregates the scraped results from the user-selected upstream search engines. /// These engines can be chosen either from the user interface (UI) or from the configuration file. @@ -37,7 +40,7 @@ type FutureVec = Vec, Report = Vec::with_capacity(0); // create tasks for upstream result fetching - let mut tasks: FutureVec = FutureVec::new(); + let tasks: FutureVec = FutureVec::new(); + let query: Arc = Arc::new(query.to_string()); for engine_handler in upstream_search_engines { - let (name, search_engine) = engine_handler.to_owned().into_name_engine(); + let (name, search_engine) = engine_handler.clone().into_name_engine(); names.push(name); - let query: String = query.to_owned(); + let query_partially_cloned = query.clone(); tasks.push(tokio::spawn(async move { search_engine - .results(&query, page, user_agent, client, safe_search) + .results( + &query_partially_cloned, + page, + user_agent, + client, + safe_search, + ) .await })); } @@ -117,7 +127,7 @@ pub async fn aggregate( } // aggregate search results, removing duplicates and handling errors the upstream engines returned - let mut result_map: HashMap = HashMap::new(); + let mut result_map: Vec<(String, SearchResult)> = Vec::new(); let mut engine_errors_info: Vec = Vec::new(); let mut handle_error = |error: &Report, engine_name: &'static str| { @@ -134,51 +144,45 @@ pub async fn aggregate( if result_map.is_empty() { match response { - Ok(results) => { - result_map = results.clone(); - } - Err(error) => { - handle_error(&error, engine); - } - } + Ok(results) => result_map = results, + Err(error) => handle_error(&error, engine), + }; continue; } match response { Ok(result) => { result.into_iter().for_each(|(key, value)| { - result_map - .entry(key) - .and_modify(|result| { - result.add_engines(engine); - }) - .or_insert_with(|| -> SearchResult { value }); + match result_map.iter().find(|(key_s, _)| key_s == &key) { + Some(value) => value.1.to_owned().add_engines(engine), + None => result_map.push((key, value)), + }; }); } - Err(error) => { - handle_error(&error, engine); - } - } + Err(error) => handle_error(&error, engine), + }; } if safe_search >= 3 { - let mut blacklist_map: HashMap = HashMap::new(); + let mut blacklist_map: Vec<(String, SearchResult)> = Vec::new(); filter_with_lists( &mut result_map, &mut blacklist_map, file_path(FileType::BlockList)?, - )?; + ) + .await?; filter_with_lists( &mut blacklist_map, &mut result_map, file_path(FileType::AllowList)?, - )?; + ) + .await?; drop(blacklist_map); } - let results: Vec = result_map.into_values().collect(); + let results: Vec = result_map.iter().map(|(_, value)| value.clone()).collect(); Ok(SearchResults::new(results, &engine_errors_info)) } @@ -187,35 +191,41 @@ pub async fn aggregate( /// /// # Arguments /// -/// * `map_to_be_filtered` - A mutable reference to a `HashMap` of search results to filter, where the filtered results will be removed from. -/// * `resultant_map` - A mutable reference to a `HashMap` to hold the filtered results. +/// * `map_to_be_filtered` - A mutable reference to a `Vec` of search results to filter, where the filtered results will be removed from. +/// * `resultant_map` - A mutable reference to a `Vec` to hold the filtered results. /// * `file_path` - A `&str` representing the path to a file containing regex patterns to use for filtering. /// /// # Errors /// /// Returns an error if the file at `file_path` cannot be opened or read, or if a regex pattern is invalid. -pub fn filter_with_lists( - map_to_be_filtered: &mut HashMap, - resultant_map: &mut HashMap, +pub async fn filter_with_lists( + map_to_be_filtered: &mut Vec<(String, SearchResult)>, + resultant_map: &mut Vec<(String, SearchResult)>, file_path: &str, ) -> Result<(), Box> { - let mut reader = BufReader::new(File::open(file_path)?); + let reader = BufReader::new(File::open(file_path).await?); + let mut lines = reader.lines(); - for line in reader.by_ref().lines() { - let re = Regex::new(line?.trim())?; + while let Some(line) = lines.next_line().await? { + let re = Regex::new(line.trim())?; + let mut length = map_to_be_filtered.len(); + let mut idx: usize = Default::default(); // Iterate over each search result in the map and check if it matches the regex pattern - for (url, search_result) in map_to_be_filtered.clone().into_iter() { - if re.is_match(&url.to_lowercase()) - || re.is_match(&search_result.title.to_lowercase()) - || re.is_match(&search_result.description.to_lowercase()) + while idx < length { + let ele = &map_to_be_filtered[idx]; + let ele_inner = &ele.1; + match re.is_match(&ele.0.to_lowercase()) + || re.is_match(&ele_inner.title.to_lowercase()) + || re.is_match(&ele_inner.description.to_lowercase()) { - // If the search result matches the regex pattern, move it from the original map to the resultant map - resultant_map.insert( - url.to_owned(), - map_to_be_filtered.remove(&url.to_owned()).unwrap(), - ); - } + true => { + // If the search result matches the regex pattern, move it from the original map to the resultant map + resultant_map.push(map_to_be_filtered.swap_remove(idx)); + length -= 1; + } + false => idx += 1, + }; } } @@ -226,15 +236,14 @@ pub fn filter_with_lists( mod tests { use super::*; use smallvec::smallvec; - use std::collections::HashMap; use std::io::Write; use tempfile::NamedTempFile; - #[test] - fn test_filter_with_lists() -> Result<(), Box> { + #[tokio::test] + async fn test_filter_with_lists() -> Result<(), Box> { // Create a map of search results to filter - let mut map_to_be_filtered = HashMap::new(); - map_to_be_filtered.insert( + let mut map_to_be_filtered = Vec::new(); + map_to_be_filtered.push(( "https://www.example.com".to_owned(), SearchResult { title: "Example Domain".to_owned(), @@ -243,15 +252,15 @@ mod tests { .to_owned(), engine: smallvec!["Google".to_owned(), "Bing".to_owned()], }, - ); - map_to_be_filtered.insert( + )); + map_to_be_filtered.push(( "https://www.rust-lang.org/".to_owned(), SearchResult { title: "Rust Programming Language".to_owned(), url: "https://www.rust-lang.org/".to_owned(), description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(), engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()], - }, + },) ); // Create a temporary file with regex patterns @@ -260,25 +269,30 @@ mod tests { writeln!(file, "rust")?; file.flush()?; - let mut resultant_map = HashMap::new(); + let mut resultant_map = Vec::new(); filter_with_lists( &mut map_to_be_filtered, &mut resultant_map, file.path().to_str().unwrap(), - )?; + ) + .await?; assert_eq!(resultant_map.len(), 2); - assert!(resultant_map.contains_key("https://www.example.com")); - assert!(resultant_map.contains_key("https://www.rust-lang.org/")); + assert!(resultant_map + .iter() + .any(|(key, _)| key == "https://www.example.com")); + assert!(resultant_map + .iter() + .any(|(key, _)| key == "https://www.rust-lang.org/")); assert_eq!(map_to_be_filtered.len(), 0); Ok(()) } - #[test] - fn test_filter_with_lists_wildcard() -> Result<(), Box> { - let mut map_to_be_filtered = HashMap::new(); - map_to_be_filtered.insert( + #[tokio::test] + async fn test_filter_with_lists_wildcard() -> Result<(), Box> { + let mut map_to_be_filtered = Vec::new(); + map_to_be_filtered.push(( "https://www.example.com".to_owned(), SearchResult { title: "Example Domain".to_owned(), @@ -287,8 +301,8 @@ mod tests { .to_owned(), engine: smallvec!["Google".to_owned(), "Bing".to_owned()], }, - ); - map_to_be_filtered.insert( + )); + map_to_be_filtered.push(( "https://www.rust-lang.org/".to_owned(), SearchResult { title: "Rust Programming Language".to_owned(), @@ -296,34 +310,39 @@ mod tests { description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(), engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()], }, - ); + )); // Create a temporary file with a regex pattern containing a wildcard let mut file = NamedTempFile::new()?; writeln!(file, "ex.*le")?; file.flush()?; - let mut resultant_map = HashMap::new(); + let mut resultant_map = Vec::new(); filter_with_lists( &mut map_to_be_filtered, &mut resultant_map, file.path().to_str().unwrap(), - )?; + ) + .await?; assert_eq!(resultant_map.len(), 1); - assert!(resultant_map.contains_key("https://www.example.com")); + assert!(resultant_map + .iter() + .any(|(key, _)| key == "https://www.example.com")); assert_eq!(map_to_be_filtered.len(), 1); - assert!(map_to_be_filtered.contains_key("https://www.rust-lang.org/")); + assert!(map_to_be_filtered + .iter() + .any(|(key, _)| key == "https://www.rust-lang.org/")); Ok(()) } - #[test] - fn test_filter_with_lists_file_not_found() { - let mut map_to_be_filtered = HashMap::new(); + #[tokio::test] + async fn test_filter_with_lists_file_not_found() { + let mut map_to_be_filtered = Vec::new(); - let mut resultant_map = HashMap::new(); + let mut resultant_map = Vec::new(); // Call the `filter_with_lists` function with a non-existent file path let result = filter_with_lists( @@ -332,13 +351,13 @@ mod tests { "non-existent-file.txt", ); - assert!(result.is_err()); + assert!(result.await.is_err()); } - #[test] - fn test_filter_with_lists_invalid_regex() { - let mut map_to_be_filtered = HashMap::new(); - map_to_be_filtered.insert( + #[tokio::test] + async fn test_filter_with_lists_invalid_regex() { + let mut map_to_be_filtered = Vec::new(); + map_to_be_filtered.push(( "https://www.example.com".to_owned(), SearchResult { title: "Example Domain".to_owned(), @@ -347,9 +366,9 @@ mod tests { .to_owned(), engine: smallvec!["Google".to_owned(), "Bing".to_owned()], }, - ); + )); - let mut resultant_map = HashMap::new(); + let mut resultant_map = Vec::new(); // Create a temporary file with an invalid regex pattern let mut file = NamedTempFile::new().unwrap(); @@ -362,6 +381,6 @@ mod tests { file.path().to_str().unwrap(), ); - assert!(result.is_err()); + assert!(result.await.is_err()); } } diff --git a/src/server/router.rs b/src/server/router.rs index c46e79d..3b78c52 100644 --- a/src/server/router.rs +++ b/src/server/router.rs @@ -7,11 +7,13 @@ use crate::{ handler::{file_path, FileType}, }; use actix_web::{get, http::header::ContentType, web, HttpRequest, HttpResponse}; -use std::fs::read_to_string; +use tokio::fs::read_to_string; /// Handles the route of index page or main page of the `websurfx` meta search engine website. #[get("/")] -pub async fn index(config: web::Data) -> Result> { +pub async fn index( + config: web::Data<&'static Config>, +) -> Result> { Ok(HttpResponse::Ok().content_type(ContentType::html()).body( crate::templates::views::index::index( &config.style.colorscheme, @@ -25,7 +27,7 @@ pub async fn index(config: web::Data) -> Result, + config: web::Data<&'static Config>, ) -> Result> { Ok(HttpResponse::Ok().content_type(ContentType::html()).body( crate::templates::views::not_found::not_found( @@ -41,7 +43,7 @@ pub async fn not_found( #[get("/robots.txt")] pub async fn robots_data(_req: HttpRequest) -> Result> { let page_content: String = - read_to_string(format!("{}/robots.txt", file_path(FileType::Theme)?))?; + read_to_string(format!("{}/robots.txt", file_path(FileType::Theme)?)).await?; Ok(HttpResponse::Ok() .content_type(ContentType::plaintext()) .body(page_content)) @@ -49,7 +51,9 @@ pub async fn robots_data(_req: HttpRequest) -> Result) -> Result> { +pub async fn about( + config: web::Data<&'static Config>, +) -> Result> { Ok(HttpResponse::Ok().content_type(ContentType::html()).body( crate::templates::views::about::about( &config.style.colorscheme, @@ -63,7 +67,7 @@ pub async fn about(config: web::Data) -> Result, + config: web::Data<&'static Config>, ) -> Result> { Ok(HttpResponse::Ok().content_type(ContentType::html()).body( crate::templates::views::settings::settings( diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index a5b2558..103b581 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -13,12 +13,12 @@ use crate::{ }; use actix_web::{get, http::header::ContentType, web, HttpRequest, HttpResponse}; use regex::Regex; -use std::{ - borrow::Cow, +use std::borrow::Cow; +use tokio::{ fs::File, - io::{BufRead, BufReader, Read}, + io::{AsyncBufReadExt, BufReader}, + join, }; -use tokio::join; /// Handles the route of search page of the `websurfx` meta search engine website and it takes /// two search url parameters `q` and `page` where `page` parameter is optional. @@ -37,8 +37,8 @@ use tokio::join; #[get("/search")] pub async fn search( req: HttpRequest, - config: web::Data, - cache: web::Data, + config: web::Data<&'static Config>, + cache: web::Data<&'static SharedCache>, ) -> Result> { use std::sync::Arc; let params = web::Query::::from_query(req.query_string())?; @@ -70,8 +70,8 @@ pub async fn search( }); search_settings.safe_search_level = get_safesearch_level( - &Some(search_settings.safe_search_level), - ¶ms.safesearch, + params.safesearch, + search_settings.safe_search_level, config.safe_search, ); @@ -158,8 +158,8 @@ pub async fn search( /// It returns the `SearchResults` struct if the search results could be successfully fetched from /// the cache or from the upstream search engines otherwise it returns an appropriate error. async fn results( - config: &Config, - cache: &web::Data, + config: &'static Config, + cache: &'static SharedCache, query: &str, page: u32, search_settings: &server_models::Cookie<'_>, @@ -188,7 +188,7 @@ async fn results( let mut results: SearchResults = SearchResults::default(); let flag: bool = - !is_match_from_filter_list(file_path(FileType::BlockList)?, query)?; + !is_match_from_filter_list(file_path(FileType::BlockList)?, query).await?; // Return early when query contains disallowed words, if flag { results.set_disallowed(); @@ -225,12 +225,12 @@ async fn results( search_results } }; - if results.engine_errors_info().is_empty() - && results.results().is_empty() - && !results.no_engines_selected() - { - results.set_filtered(); - } + let (engine_errors_info, results_empty_check, no_engines_selected) = ( + results.engine_errors_info().is_empty(), + results.results().is_empty(), + results.no_engines_selected(), + ); + results.set_filtered(engine_errors_info & results_empty_check & !no_engines_selected); cache .cache_results(&[results.clone()], &[cache_key.clone()]) .await?; @@ -252,13 +252,14 @@ async fn results( /// /// Returns a bool indicating whether the results were found in the list or not on success /// otherwise returns a standard error type on a failure. -fn is_match_from_filter_list( +async fn is_match_from_filter_list( file_path: &str, query: &str, ) -> Result> { - let mut reader = BufReader::new(File::open(file_path)?); - for line in reader.by_ref().lines() { - let re = Regex::new(&line?)?; + let reader = BufReader::new(File::open(file_path).await?); + let mut lines = reader.lines(); + while let Some(line) = lines.next_line().await? { + let re = Regex::new(&line)?; if re.is_match(query) { return Ok(true); } @@ -267,24 +268,95 @@ fn is_match_from_filter_list( Ok(false) } -/// A helper function to modify the safe search level based on the url params. -/// The `safe_search` is the one in the user's cookie or -/// the default set by the server config if the cookie was missing. +/// A helper function to choose the safe search level value based on the URL parameters, +/// cookie value and config value. /// /// # Argurments /// -/// * `url_level` - Safe search level from the url. -/// * `safe_search` - User's cookie, or the safe search level set by the server -/// * `config_level` - Safe search level to fall back to -fn get_safesearch_level(cookie_level: &Option, url_level: &Option, config_level: u8) -> u8 { - match url_level { - Some(url_level) => { - if *url_level >= 3 { - config_level - } else { - *url_level - } +/// * `safe_search_level_from_url` - Safe search level from the URL parameters. +/// * `cookie_safe_search_level` - Safe search level value from the cookie. +/// * `config_safe_search_level` - Safe search level value from the config file. +/// +/// # Returns +/// +/// Returns an appropriate safe search level value based on the safe search level values +/// from the URL parameters, cookie and the config file. +fn get_safesearch_level( + safe_search_level_from_url: Option, + cookie_safe_search_level: u8, + config_safe_search_level: u8, +) -> u8 { + (u8::from(safe_search_level_from_url.is_some()) + * ((u8::from(config_safe_search_level >= 3) * config_safe_search_level) + + (u8::from(config_safe_search_level < 3) * safe_search_level_from_url.unwrap_or(0)))) + + (u8::from(safe_search_level_from_url.is_none()) + * ((u8::from(config_safe_search_level >= 3) * config_safe_search_level) + + (u8::from(config_safe_search_level < 3) * cookie_safe_search_level))) +} + +#[cfg(test)] +mod tests { + use std::time::{SystemTime, UNIX_EPOCH}; + + /// A helper function which creates a random mock safe search level value. + /// + /// # Returns + /// + /// Returns an optional u8 value. + fn mock_safe_search_level_value() -> Option { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .subsec_nanos() as f32; + let delay = ((nanos / 1_0000_0000 as f32).floor() as i8) - 1; + + match delay { + -1 => None, + some_num => Some(if some_num > 4 { some_num - 4 } else { some_num } as u8), } - None => cookie_level.unwrap_or(config_level), + } + + #[test] + /// A test function to test whether the output of the branchless and branched code + /// for the code to choose the appropriate safe search level is same or not. + fn get_safesearch_level_branched_branchless_code_test() { + // Get mock values for the safe search level values for URL parameters, cookie + // and config. + let safe_search_level_from_url = mock_safe_search_level_value(); + let cookie_safe_search_level = mock_safe_search_level_value().unwrap_or(0); + let config_safe_search_level = mock_safe_search_level_value().unwrap_or(0); + + // Branched code + let safe_search_level_value_from_branched_code = match safe_search_level_from_url { + Some(safe_search_level_from_url_parsed) => { + if config_safe_search_level >= 3 { + config_safe_search_level + } else { + safe_search_level_from_url_parsed + } + } + None => { + if config_safe_search_level >= 3 { + config_safe_search_level + } else { + cookie_safe_search_level + } + } + }; + + // branchless code + let safe_search_level_value_from_branchless_code = + (u8::from(safe_search_level_from_url.is_some()) + * ((u8::from(config_safe_search_level >= 3) * config_safe_search_level) + + (u8::from(config_safe_search_level < 3) + * safe_search_level_from_url.unwrap_or(0)))) + + (u8::from(safe_search_level_from_url.is_none()) + * ((u8::from(config_safe_search_level >= 3) * config_safe_search_level) + + (u8::from(config_safe_search_level < 3) * cookie_safe_search_level))); + + assert_eq!( + safe_search_level_value_from_branched_code, + safe_search_level_value_from_branchless_code + ); } } diff --git a/src/templates/partials/settings_tabs/engines.rs b/src/templates/partials/settings_tabs/engines.rs index 748cc07..de67ac7 100644 --- a/src/templates/partials/settings_tabs/engines.rs +++ b/src/templates/partials/settings_tabs/engines.rs @@ -55,7 +55,7 @@ pub fn engines(engine_names: &HashMap) -> Markup { input type="checkbox" class="engine" checked; span class="slider round"{} } - (format!("{}{}",engine_name[..1].to_uppercase().to_owned(), engine_name[1..].to_owned())) + (format!("{}{}",&engine_name[..1].to_uppercase(), &engine_name[1..])) } } @else { @@ -64,7 +64,7 @@ pub fn engines(engine_names: &HashMap) -> Markup { input type="checkbox" class="engine"; span class="slider round"{} } - (format!("{}{}",engine_name[..1].to_uppercase().to_owned(), engine_name[1..].to_owned())) + (format!("{}{}",&engine_name[..1], &engine_name[1..])) } } } diff --git a/src/templates/partials/settings_tabs/user_interface.rs b/src/templates/partials/settings_tabs/user_interface.rs index 6916b26..8f685be 100644 --- a/src/templates/partials/settings_tabs/user_interface.rs +++ b/src/templates/partials/settings_tabs/user_interface.rs @@ -36,7 +36,7 @@ fn style_option_list( } if style_type == "animations" { - style_option_names.push(("".to_owned(), "none".to_owned())) + style_option_names.push((String::default(), "none".to_owned())) } Ok(style_option_names) @@ -83,9 +83,11 @@ pub fn user_interface( "Select the animation for your theme to be used in user interface" } select name="animations"{ + @let default_animation = &String::default(); + @let animation = animation.as_ref().unwrap_or(default_animation); // Sets the user selected animation name from the config file as the first option in the selection list. - option value=(animation.as_ref().unwrap_or(&"".to_owned())){(animation.as_ref().unwrap_or(&"".to_owned()).replace('-'," "))} - @for (k,v) in style_option_list("animations", animation.as_ref().unwrap_or(&"".to_owned()))?{ + option value=(animation){(animation.replace('-'," "))} + @for (k,v) in style_option_list("animations", animation)?{ option value=(k){(v)} } } diff --git a/src/templates/views/search.rs b/src/templates/views/search.rs index 07b8b42..c5ab456 100644 --- a/src/templates/views/search.rs +++ b/src/templates/views/search.rs @@ -38,7 +38,7 @@ pub fn search( small{(result.url)} p{(PreEscaped(&result.description))} .upstream_engines{ - @for name in result.clone().engine{ + @for name in &result.engine { span{(name)} } } diff --git a/tests/index.rs b/tests/index.rs index 563c2d9..010795d 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -1,14 +1,17 @@ -use std::net::TcpListener; +use std::{net::TcpListener, sync::OnceLock}; use websurfx::{config::parser::Config, run, templates::views}; +/// A static constant for holding the parsed config. +static CONFIG: OnceLock = OnceLock::new(); + // Starts a new instance of the HTTP server, bound to a random available port async fn spawn_app() -> String { // Binding to port 0 will trigger the OS to assign a port for us. let listener = TcpListener::bind("127.0.0.1:0").expect("Failed to bind random port"); let port = listener.local_addr().unwrap().port(); - let config = Config::parse(false).unwrap(); - let cache = websurfx::cache::cacher::create_cache(&config).await; + let config = CONFIG.get_or_init(|| Config::parse(false).unwrap()); + let cache = websurfx::cache::cacher::create_cache(config).await; let server = run(listener, config, cache).expect("Failed to bind address"); tokio::spawn(server);