diff --git a/.cspell.json b/.cspell.json index b798d29..9d5ec5b 100644 --- a/.cspell.json +++ b/.cspell.json @@ -11,6 +11,10 @@ ], "language": "en", "noConfigSearch": true, - "words": ["megalinter", "oxsecurity"], + "words": [ + "megalinter", + "oxsecurity", + "websurfx" + ], "version": "0.2" } diff --git a/Cargo.lock b/Cargo.lock index aac4c9a..61a2d0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ dependencies = [ "futures-sink", "memchr", "pin-project-lite", - "tokio 1.29.1", + "tokio 1.32.0", "tokio-util", "tracing", ] @@ -90,7 +90,7 @@ dependencies = [ "rand 0.8.5", "sha1", "smallvec 1.11.0", - "tokio 1.29.1", + "tokio 1.32.0", "tokio-util", "tracing", "zstd", @@ -102,8 +102,8 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ - "quote 1.0.32", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -126,7 +126,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" dependencies = [ "futures-core", - "tokio 1.29.1", + "tokio 1.32.0", ] [[package]] @@ -142,8 +142,8 @@ dependencies = [ "futures-util", "mio 0.8.8", "num_cpus", - "socket2", - "tokio 1.29.1", + "socket2 0.4.9", + "tokio 1.32.0", "tracing", ] @@ -204,7 +204,7 @@ dependencies = [ "serde_json", "serde_urlencoded 0.7.1", "smallvec 1.11.0", - "socket2", + "socket2 0.4.9", "time 0.3.25", "url 2.4.0", ] @@ -217,7 +217,7 @@ checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" dependencies = [ "actix-router", "proc-macro2 1.0.66", - "quote 1.0.32", + "quote 1.0.33", "syn 1.0.109", ] @@ -261,9 +261,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" dependencies = [ "memchr", ] @@ -297,9 +297,9 @@ checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" [[package]] name = "anyhow" -version = "1.0.72" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "askama_escape" @@ -309,13 +309,13 @@ checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" [[package]] name = "async-trait" -version = "0.1.72" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.32", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -386,9 +386,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "block-buffer" @@ -475,9 +475,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c6b2562119bf28c3439f7f02db99faf0aa1a8cdfe5772a2ee155d32227239f0" +checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" dependencies = [ "jobserver", "libc", @@ -533,18 +533,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.19" +version = "4.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d" +checksum = "03aef18ddf7d879c15ce20f04826ef8418101c7e528014c3eeea13321047dca3" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.3.19" +version = "4.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1" +checksum = "f8ce6fffb678c9b80a70b6b6de0aad31df727623a70fd9a842c30cd573e2fa98" dependencies = [ "anstyle", "clap_lex", @@ -810,8 +810,8 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ - "quote 1.0.32", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -828,7 +828,7 @@ checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", "proc-macro2 1.0.66", - "quote 1.0.32", + "quote 1.0.33", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -956,7 +956,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.32", + "quote 1.0.33", "syn 1.0.109", "synstructure", ] @@ -980,9 +980,9 @@ checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" dependencies = [ "crc32fast", "miniz_oxide", @@ -1188,7 +1188,7 @@ dependencies = [ "http 0.2.9", "indexmap", "slab", - "tokio 1.29.1", + "tokio 1.32.0", "tokio-util", "tracing", ] @@ -1250,7 +1250,7 @@ dependencies = [ "mac", "markup5ever 0.11.0", "proc-macro2 1.0.66", - "quote 1.0.32", + "quote 1.0.33", "syn 1.0.109", ] @@ -1313,9 +1313,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" @@ -1370,8 +1370,8 @@ dependencies = [ "httpdate", "itoa 1.0.9", "pin-project-lite", - "socket2", - "tokio 1.29.1", + "socket2 0.4.9", + "tokio 1.32.0", "tower-service", "tracing", "want 0.3.1", @@ -1399,7 +1399,7 @@ dependencies = [ "bytes 1.4.0", "hyper 0.14.27", "native-tls", - "tokio 1.29.1", + "tokio 1.32.0", "tokio-native-tls", ] @@ -1583,9 +1583,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "mac" @@ -1837,8 +1837,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.32", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -1955,8 +1955,8 @@ dependencies = [ "pest", "pest_meta", "proc-macro2 1.0.66", - "quote 1.0.32", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -2057,8 +2057,8 @@ dependencies = [ "phf_generator 0.11.2", "phf_shared 0.11.2", "proc-macro2 1.0.66", - "quote 1.0.32", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -2090,9 +2090,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c516611246607d0c04186886dbb3a754368ef82c79e9827a802c6d836dd111c" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" [[package]] name = "pin-utils" @@ -2185,9 +2185,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2 1.0.66", ] @@ -2361,16 +2361,16 @@ dependencies = [ [[package]] name = "redis" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff5d95dd18a4d76650f0c2607ed8ebdbf63baf9cb934e1c233cd220c694db1d7" +checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02" dependencies = [ "combine", "itoa 1.0.9", "percent-encoding 2.3.0", "ryu", "sha1_smol", - "socket2", + "socket2 0.4.9", "url 2.4.0", ] @@ -2479,7 +2479,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.1", - "tokio 1.29.1", + "tokio 1.32.0", "tokio-native-tls", "tower-service", "url 2.4.0", @@ -2539,11 +2539,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.7" +version = "0.38.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399" +checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", "errno", "libc", "linux-raw-sys", @@ -2648,7 +2648,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", "cssparser", "derive_more", "fxhash", @@ -2684,29 +2684,29 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.182" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb30a74471f5b7a1fa299f40b4bf1be93af61116df95465b2b5fc419331e430" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.182" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f4c2c6ea4bc09b5c419012eafcdb0fcef1d9119d626c8f3a0708a5b92d38a70" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.32", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] name = "serde_json" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" +checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" dependencies = [ "itoa 1.0.9", "ryu", @@ -2829,6 +2829,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2882,7 +2892,7 @@ dependencies = [ "phf_generator 0.7.24", "phf_shared 0.7.24", "proc-macro2 1.0.66", - "quote 1.0.32", + "quote 1.0.33", "string_cache_shared", ] @@ -2895,7 +2905,7 @@ dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", "proc-macro2 1.0.66", - "quote 1.0.32", + "quote 1.0.33", ] [[package]] @@ -2922,18 +2932,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.32", + "quote 1.0.33", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.28" +version = "2.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.32", + "quote 1.0.33", "unicode-ident", ] @@ -2944,16 +2954,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.32", + "quote 1.0.33", "syn 1.0.109", "unicode-xid 0.2.4", ] [[package]] name = "tempfile" -version = "3.7.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if 1.0.0", "fastrand", @@ -2984,22 +2994,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.44" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.44" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.32", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -3087,11 +3097,10 @@ dependencies = [ [[package]] name = "tokio" -version = "1.29.1" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg 1.1.0", "backtrace", "bytes 1.4.0", "libc", @@ -3100,7 +3109,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.3", "tokio-macros", "windows-sys", ] @@ -3154,8 +3163,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.32", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -3165,7 +3174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.29.1", + "tokio 1.32.0", ] [[package]] @@ -3250,7 +3259,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.29.1", + "tokio 1.32.0", "tracing", ] @@ -3476,8 +3485,8 @@ dependencies = [ "log", "once_cell", "proc-macro2 1.0.66", - "quote 1.0.32", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", "wasm-bindgen-shared", ] @@ -3499,7 +3508,7 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ - "quote 1.0.32", + "quote 1.0.33", "wasm-bindgen-macro-support", ] @@ -3510,8 +3519,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.32", - "syn 2.0.28", + "quote 1.0.33", + "syn 2.0.29", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3534,7 +3543,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.16.5" +version = "0.17.0" dependencies = [ "actix-cors", "actix-files", @@ -3556,7 +3565,7 @@ dependencies = [ "scraper", "serde", "serde_json", - "tokio 1.29.1", + "tokio 1.32.0", ] [[package]] @@ -3613,9 +3622,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -3628,45 +3637,45 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winreg" diff --git a/Cargo.toml b/Cargo.toml index 866f3ca..f83aa88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,31 +1,31 @@ [package] name = "websurfx" -version = "0.16.5" +version = "0.17.0" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" license = "AGPL-3.0" [dependencies] -reqwest = {version="*",features=["json"]} -tokio = {version="*",features=["full"]} -serde = {version="*",features=["derive"]} -handlebars = { version = "4.3.6", features = ["dir_source"] } -scraper = {version="*"} +reqwest = {version="0.11.18",features=["json"]} +tokio = {version="1.32.0",features=["full"]} +serde = {version="1.0.183",features=["derive"]} +handlebars = { version = "4.3.7", features = ["dir_source"] } +scraper = {version="0.17.1"} actix-web = {version="4.3.1", features = ["cookies"]} actix-files = {version="0.6.2"} actix-cors = {version="0.6.4"} -serde_json = {version="*"} -fake-useragent = {version="*"} +serde_json = {version="1.0.105"} +fake-useragent = {version="0.1.3"} env_logger = {version="0.10.0"} -log = {version="0.4.17"} -rlua = {version="*"} -redis = {version="*"} -md5 = {version="*"} -rand={version="*"} -once_cell = {version="*"} +log = {version="0.4.20"} +rlua = {version="0.19.7"} +redis = {version="0.23.2"} +md5 = {version="0.7.0"} +rand={version="0.8.5"} +once_cell = {version="1.18.0"} error-stack = {version="0.3.1"} -async-trait = {version="*"} +async-trait = {version="0.1.73"} [dev-dependencies] rusty-hook = "^0.11.2" diff --git a/public/templates/search.html b/public/templates/search.html index 7aeabe9..ff771f1 100644 --- a/public/templates/search.html +++ b/public/templates/search.html @@ -4,7 +4,7 @@
{{#each results}}
-

{{{this.title}}}

+

{{{this.title}}}

{{{this.url}}}

{{{this.description}}}

diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index 6807749..75d0b8d 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -15,7 +15,7 @@ use websurfx::{config::parser::Config, run}; #[actix_web::main] async fn main() -> std::io::Result<()> { // Initialize the parsed config file. - let config = Config::parse(true).unwrap(); + let config = Config::parse(false).unwrap(); log::info!( "started server on port {} and IP {}", diff --git a/src/config/parser.rs b/src/config/parser.rs index 201e579..e05f56d 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -34,7 +34,7 @@ pub struct Config { pub aggregator: AggregatorConfig, pub logging: bool, pub debug: bool, - pub upstream_search_engines: Vec, + pub upstream_search_engines: Vec, pub request_timeout: u8, pub threads: u8, } @@ -57,7 +57,7 @@ impl Config { /// # Arguments /// /// * `logging_initialized` - It takes a boolean which ensures that the logging doesn't get - /// initialized twice. + /// initialized twice. Pass false if the logger has not yet been initialized. /// /// # Error /// @@ -77,22 +77,8 @@ impl Config { let debug: bool = globals.get::<_, bool>("debug")?; let logging:bool= globals.get::<_, bool>("logging")?; - // Check whether logging has not been initialized before. - if logging_initialized { - if let Ok(pkg_env_var) = std::env::var("PKG_ENV"){ - if pkg_env_var.to_lowercase() == "dev" { - env_logger::Builder::new().filter(None, LevelFilter::Trace).init(); - } - } else { - // Initializing logging middleware with level set to default or info. - let mut log_level: LevelFilter = LevelFilter::Error; - if logging && debug == false { - log_level = LevelFilter::Info; - } else if debug { - log_level = LevelFilter::Debug; - }; - env_logger::Builder::new().filter(None, log_level).init(); - } + if !logging_initialized { + set_logging_level(debug, logging); } let threads: u8 = if parsed_threads == 0 { @@ -121,12 +107,14 @@ impl Config { .get::<_, HashMap>("upstream_search_engines")? .into_iter() .filter_map(|(key, value)| value.then_some(key)) + .filter_map(|engine| crate::engines::engine_models::EngineHandler::new(&engine)) .collect(), request_timeout: globals.get::<_, u8>("request_timeout")?, threads, }) }) } + /// A helper function which returns an appropriate config file path checking if the config /// file exists on that path. /// @@ -173,3 +161,25 @@ impl Config { Err("Config file not found!!".to_string().into()) } } + +/// a helper function that sets the proper logging level +fn set_logging_level(debug: bool, logging: bool) { + if let Ok(pkg_env_var) = std::env::var("PKG_ENV") { + if pkg_env_var.to_lowercase() == "dev" { + env_logger::Builder::new() + .filter(None, LevelFilter::Trace) + .init(); + return; + } + } + + // Initializing logging middleware with level set to default or info. + let log_level = match (debug, logging) { + (true, true) => LevelFilter::Debug, + (true, false) => LevelFilter::Debug, + (false, true) => LevelFilter::Info, + (false, false) => LevelFilter::Error, + }; + + env_logger::Builder::new().filter(None, log_level).init(); +} diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index 42f4994..11b7d86 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT}; use scraper::{Html, Selector}; -use crate::results::aggregation_models::RawSearchResult; +use crate::results::aggregation_models::SearchResult; use super::engine_models::{EngineError, SearchEngine}; @@ -43,7 +43,7 @@ impl SearchEngine for DuckDuckGo { page: u32, user_agent: String, request_timeout: u8, - ) -> Result, EngineError> { + ) -> Result, EngineError> { // Page number can be missing or empty string and so appropriate handling is required // so that upstream server recieves valid page number. let url: String = match page { @@ -120,7 +120,7 @@ impl SearchEngine for DuckDuckGo { Ok(document .select(&results) .map(|result| { - RawSearchResult::new( + SearchResult::new( result .select(&result_title) .next() @@ -147,7 +147,7 @@ impl SearchEngine for DuckDuckGo { vec!["duckduckgo".to_string()], ) }) - .map(|search_result| (search_result.visiting_url.clone(), search_result)) + .map(|search_result| (search_result.url.clone(), search_result)) .collect()) } } diff --git a/src/engines/engine_models.rs b/src/engines/engine_models.rs index b5051be..d33d13c 100644 --- a/src/engines/engine_models.rs +++ b/src/engines/engine_models.rs @@ -1,7 +1,7 @@ //! This module provides the error enum to handle different errors associated while requesting data from //! the upstream search engines with the search query provided by the user. -use crate::results::aggregation_models::RawSearchResult; +use crate::results::aggregation_models::SearchResult; use error_stack::{IntoReport, Result, ResultExt}; use std::{collections::HashMap, fmt, time::Duration}; @@ -43,9 +43,9 @@ impl fmt::Display for EngineError { impl error_stack::Context for EngineError {} -/// A trait to define common behaviour for all search engines. +/// A trait to define common behavior for all search engines. #[async_trait::async_trait] -pub trait SearchEngine { +pub trait SearchEngine: Sync + Send { async fn fetch_html_from_upstream( &self, url: String, @@ -56,7 +56,7 @@ pub trait SearchEngine { Ok(reqwest::Client::new() .get(url) .timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server - .headers(header_map) // add spoofed headers to emulate human behaviour + .headers(header_map) // add spoofed headers to emulate human behavior .send() .await .into_report() @@ -73,5 +73,37 @@ pub trait SearchEngine { page: u32, user_agent: String, request_timeout: u8, - ) -> Result, EngineError>; + ) -> Result, EngineError>; +} + +pub struct EngineHandler { + engine: Box, + name: &'static str, +} + +impl Clone for EngineHandler { + fn clone(&self) -> Self { + Self::new(self.name).unwrap() + } +} + +impl EngineHandler { + /// parses an engine name into an engine handler, returns none if the engine is unknown + pub fn new(engine_name: &str) -> Option { + let engine: (&'static str, Box) = + match engine_name.to_lowercase().as_str() { + "duckduckgo" => ("duckduckgo", Box::new(super::duckduckgo::DuckDuckGo)), + "searx" => ("searx", Box::new(super::searx::Searx)), + _ => return None, + }; + + Some(Self { + engine: engine.1, + name: engine.0, + }) + } + + pub fn into_name_engine(self) -> (&'static str, Box) { + (self.name, self.engine) + } } diff --git a/src/engines/searx.rs b/src/engines/searx.rs index ab73ee0..4ad41f5 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -6,7 +6,7 @@ use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT}; use scraper::{Html, Selector}; use std::collections::HashMap; -use crate::results::aggregation_models::RawSearchResult; +use crate::results::aggregation_models::SearchResult; use super::engine_models::{EngineError, SearchEngine}; use error_stack::{IntoReport, Report, Result, ResultExt}; @@ -42,7 +42,7 @@ impl SearchEngine for Searx { page: u32, user_agent: String, request_timeout: u8, - ) -> Result, EngineError> { + ) -> Result, EngineError> { // Page number can be missing or empty string and so appropriate handling is required // so that upstream server recieves valid page number. let url: String = match page { @@ -111,7 +111,7 @@ impl SearchEngine for Searx { Ok(document .select(&results) .map(|result| { - RawSearchResult::new( + SearchResult::new( result .select(&result_title) .next() @@ -137,7 +137,7 @@ impl SearchEngine for Searx { vec!["searx".to_string()], ) }) - .map(|search_result| (search_result.visiting_url.clone(), search_result)) + .map(|search_result| (search_result.url.clone(), search_result)) .collect()) } } diff --git a/src/results/aggregation_models.rs b/src/results/aggregation_models.rs index 6766fae..b4a780c 100644 --- a/src/results/aggregation_models.rs +++ b/src/results/aggregation_models.rs @@ -5,55 +5,6 @@ use serde::{Deserialize, Serialize}; use crate::{config::parser_models::Style, engines::engine_models::EngineError}; -/// A named struct to store, serialize and deserializes the individual search result from all the -/// scraped and aggregated search results from the upstream search engines. -/// -/// # Fields -/// -/// * `title` - The title of the search result. -/// * `visiting_url` - The url which is accessed when clicked on it (href url in html in simple -/// words). -/// * `url` - The url to be displayed below the search result title in html. -/// * `description` - The description of the search result. -/// * `engine` - The names of the upstream engines from which this results were provided. -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SearchResult { - pub title: String, - pub visiting_url: String, - pub url: String, - pub description: String, - pub engine: Vec, -} - -impl SearchResult { - /// Constructs a new `SearchResult` with the given arguments needed for the struct. - /// - /// # Arguments - /// - /// * `title` - The title of the search result. - /// * `visiting_url` - The url which is accessed when clicked on it - /// (href url in html in simple words). - /// * `url` - The url to be displayed below the search result title in html. - /// * `description` - The description of the search result. - /// * `engine` - The names of the upstream engines from which this results were provided. - pub fn new( - title: String, - visiting_url: String, - url: String, - description: String, - engine: Vec, - ) -> Self { - SearchResult { - title, - visiting_url, - url, - description, - engine, - } - } -} - /// A named struct to store the raw scraped search results scraped search results from the /// upstream search engines before aggregating it.It derives the Clone trait which is needed /// to write idiomatic rust using `Iterators`. @@ -61,37 +12,33 @@ impl SearchResult { /// # Fields /// /// * `title` - The title of the search result. -/// * `visiting_url` - The url which is accessed when clicked on it +/// * `url` - The url which is accessed when clicked on it /// (href url in html in simple words). /// * `description` - The description of the search result. /// * `engine` - The names of the upstream engines from which this results were provided. -#[derive(Clone)] -pub struct RawSearchResult { +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchResult { pub title: String, - pub visiting_url: String, + pub url: String, pub description: String, pub engine: Vec, } -impl RawSearchResult { +impl SearchResult { /// Constructs a new `RawSearchResult` with the given arguments needed for the struct. /// /// # Arguments /// /// * `title` - The title of the search result. - /// * `visiting_url` - The url which is accessed when clicked on it + /// * `url` - The url which is accessed when clicked on it /// (href url in html in simple words). /// * `description` - The description of the search result. /// * `engine` - The names of the upstream engines from which this results were provided. - pub fn new( - title: String, - visiting_url: String, - description: String, - engine: Vec, - ) -> Self { - RawSearchResult { + pub fn new(title: String, url: String, description: String, engine: Vec) -> Self { + SearchResult { title, - visiting_url, + url, description, engine, } diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index 4ffaaf5..16586c0 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -8,18 +8,14 @@ use rand::Rng; use tokio::task::JoinHandle; use super::{ - aggregation_models::{EngineErrorInfo, RawSearchResult, SearchResult, SearchResults}, + aggregation_models::{EngineErrorInfo, SearchResult, SearchResults}, user_agent::random_user_agent, }; -use crate::engines::{ - duckduckgo, - engine_models::{EngineError, SearchEngine}, - searx, -}; +use crate::engines::engine_models::{EngineError, EngineHandler}; /// Aliases for long type annotations -type FutureVec = Vec, Report>>>; +type FutureVec = Vec, Report>>>; /// The function aggregates the scraped results from the user-selected upstream search engines. /// These engines can be chosen either from the user interface (UI) or from the configuration file. @@ -64,139 +60,93 @@ pub async fn aggregate( page: u32, random_delay: bool, debug: bool, - upstream_search_engines: Vec, + upstream_search_engines: Vec, request_timeout: u8, ) -> Result> { let user_agent: String = random_user_agent(); - let mut result_map: HashMap = HashMap::new(); // Add a random delay before making the request. if random_delay || !debug { let mut rng = rand::thread_rng(); let delay_secs = rng.gen_range(1..10); - std::thread::sleep(Duration::from_secs(delay_secs)); + tokio::time::sleep(Duration::from_secs(delay_secs)).await; } - // fetch results from upstream search engines simultaneously/concurrently. - let search_engines: Vec> = upstream_search_engines - .iter() - .map(|engine| match engine.to_lowercase().as_str() { - "duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box, - "searx" => Box::new(searx::Searx) as Box, - &_ => panic!("Config Error: Incorrect config file option provided"), - }) - .collect(); + let mut names: Vec<&str> = vec![]; - let task_capacity: usize = search_engines.len(); + // create tasks for upstream result fetching + let mut tasks: FutureVec = FutureVec::new(); - let tasks: FutureVec = search_engines - .into_iter() - .map(|search_engine| { - let query: String = query.clone(); - let user_agent: String = user_agent.clone(); - tokio::spawn(async move { - search_engine - .results(query, page, user_agent.clone(), request_timeout) - .await - }) - }) - .collect(); + for engine_handler in upstream_search_engines { + let (name, search_engine) = engine_handler.into_name_engine(); + names.push(name); + let query: String = query.clone(); + let user_agent: String = user_agent.clone(); + tasks.push(tokio::spawn(async move { + search_engine + .results(query, page, user_agent.clone(), request_timeout) + .await + })); + } - let mut outputs = Vec::with_capacity(task_capacity); + // get upstream responses + let mut responses = Vec::with_capacity(tasks.len()); for task in tasks { if let Ok(result) = task.await { - outputs.push(result) + responses.push(result) } } + // aggregate search results, removing duplicates and handling errors the upstream engines returned + let mut result_map: HashMap = HashMap::new(); let mut engine_errors_info: Vec = Vec::new(); - // The code block `outputs.iter()` determines whether it is the first time the code is being run. - // It does this by checking the initial flag. If it is the first time, the code selects the first - // engine from which results are fetched and adds or extends them into the `result_map`. If the - // initially selected engine fails, the code automatically selects another engine to map or extend - // into the `result_map`. On the other hand, if an engine selected for the first time successfully - // fetches results and maps them into the `result_map`, the initial flag is set to false. Subsequently, - // the code iterates through the remaining engines one by one. It compares the fetched results from each - // engine with the results already present in the `result_map` to identify any duplicates. If duplicate - // results are found, the code groups them together with the name of the engine from which they were - // fetched, and automatically removes the duplicate results from the newly fetched data. - // - // Additionally, the code handles errors returned by the engines. It keeps track of which engines - // encountered errors and stores this information in a vector of structures called `EngineErrorInfo`. - // Each structure in this vector contains the name of the engine and the type of error it returned. - // These structures will later be added to the final `SearchResults` structure. The `SearchResults` - // structure is used to display an error box in the UI containing the relevant information from - // the `EngineErrorInfo` structure. - // - // In summary, this code block manages the selection of engines, handling of duplicate results, and tracking - // of errors in order to populate the `result_map` and provide informative feedback to the user through the - // `SearchResults` structure. - let mut initial: bool = true; - let mut counter: usize = 0; - outputs.iter().for_each(|results| { - if initial { - match results { - Ok(result) => { - result_map.extend(result.clone()); - counter += 1; - initial = false + let mut handle_error = |error: Report, engine_name: String| { + log::error!("Engine Error: {:?}", error); + engine_errors_info.push(EngineErrorInfo::new( + error.downcast_ref::().unwrap(), + engine_name.to_string(), + )); + }; + + for _ in 0..responses.len() { + let response = responses.pop().unwrap(); + let engine = names.pop().unwrap().to_string(); + + if result_map.is_empty() { + match response { + Ok(results) => { + result_map = results.clone(); } - Err(error_type) => { - log::error!("Engine Error: {:?}", error_type); - engine_errors_info.push(EngineErrorInfo::new( - error_type.downcast_ref::().unwrap(), - upstream_search_engines[counter].clone(), - )); - counter += 1 + Err(error) => { + handle_error(error, engine); } } - } else { - match results { - Ok(result) => { - result.clone().into_iter().for_each(|(key, value)| { - result_map - .entry(key) - .and_modify(|result| { - result.add_engines(value.clone().engine()); - }) - .or_insert_with(|| -> RawSearchResult { - RawSearchResult::new( - value.title.clone(), - value.visiting_url.clone(), - value.description.clone(), - value.engine.clone(), - ) - }); - }); - counter += 1 - } - Err(error_type) => { - log::error!("Engine Error: {:?}", error_type); - engine_errors_info.push(EngineErrorInfo::new( - error_type.downcast_ref::().unwrap(), - upstream_search_engines[counter].clone(), - )); - counter += 1 - } + continue; + } + + match response { + Ok(result) => { + result.into_iter().for_each(|(key, value)| { + result_map + .entry(key) + .and_modify(|result| { + result.add_engines(engine.clone()); + }) + .or_insert_with(|| -> SearchResult { value }); + }); + } + Err(error) => { + handle_error(error, engine); } } - }); + } + + let results = result_map.into_values().collect(); Ok(SearchResults::new( - result_map - .into_iter() - .map(|(key, value)| { - SearchResult::new( - value.title, - value.visiting_url, - key, - value.description, - value.engine, - ) - }) - .collect(), + results, query.to_string(), engine_errors_info, )) diff --git a/src/server/routes.rs b/src/server/routes.rs index c203e20..77210b2 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -7,6 +7,7 @@ use std::fs::read_to_string; use crate::{ cache::cacher::RedisCache, config::parser::Config, + engines::engine_models::EngineHandler, handler::public_paths::public_path, results::{aggregation_models::SearchResults, aggregator::aggregate}, }; @@ -175,12 +176,19 @@ async fn results( { Some(cookie_value) => { let cookie_value: Cookie = serde_json::from_str(cookie_value.name_value().1)?; + + let engines = cookie_value + .engines + .iter() + .filter_map(|name| EngineHandler::new(name)) + .collect(); + aggregate( query, page, config.aggregator.random_delay, config.debug, - cookie_value.engines, + engines, config.request_timeout, ) .await? diff --git a/tests/index.rs b/tests/index.rs index d886e13..080ad27 100644 --- a/tests/index.rs +++ b/tests/index.rs @@ -8,7 +8,7 @@ fn spawn_app() -> String { // Binding to port 0 will trigger the OS to assign a port for us. let listener = TcpListener::bind("127.0.0.1:0").expect("Failed to bind random port"); let port = listener.local_addr().unwrap().port(); - let config = Config::parse(true).unwrap(); + let config = Config::parse(false).unwrap(); let server = run(listener, config).expect("Failed to bind address"); tokio::spawn(server); @@ -36,7 +36,7 @@ async fn test_index() { assert_eq!(res.status(), 200); let handlebars = handlebars(); - let config = Config::parse(false).unwrap(); + let config = Config::parse(true).unwrap(); let template = handlebars.render("index", &config.style).unwrap(); assert_eq!(res.text().await.unwrap(), template); }