alamin/websurfx
alamin/websurfx
0
0
Fork 0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-24 15:08:22 -05:00
Code Issues Releases Activity

Merge branch 'rolling' into feat-error-box-for-engine-errors

Browse Source
This commit is contained in:
zhou fan 2023-08-24 08:02:03 +08:00 committed by GitHub
commit 575a7f95ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 344 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.cspell.json
View File

@ -11,6 +11,10 @@
], ],
"language": "en", "language": "en",
"noConfigSearch": true, "noConfigSearch": true,
"words": ["megalinter", "oxsecurity"], "words": [
"megalinter",
"oxsecurity",
"websurfx"
],
"version": "0.2" "version": "0.2"
} }

241
Cargo.lock generated
View File

@ -14,7 +14,7 @@ dependencies = [
"futures-sink", "futures-sink",
"memchr", "memchr",
"pin-project-lite", "pin-project-lite",
"tokio 1.29.1", "tokio 1.32.0",
"tokio-util", "tokio-util",
"tracing", "tracing",
] ]
@ -90,7 +90,7 @@ dependencies = [
"rand 0.8.5", "rand 0.8.5",
"sha1", "sha1",
"smallvec 1.11.0", "smallvec 1.11.0",
"tokio 1.29.1", "tokio 1.32.0",
"tokio-util", "tokio-util",
"tracing", "tracing",
"zstd", "zstd",
@ -102,8 +102,8 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
dependencies = [ dependencies = [
"quote 1.0.32", "quote 1.0.33",
"syn 2.0.28", "syn 2.0.29",
] ]
[[package]] [[package]]
@ -126,7 +126,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"tokio 1.29.1", "tokio 1.32.0",
] ]
[[package]] [[package]]
@ -142,8 +142,8 @@ dependencies = [
"futures-util", "futures-util",
"mio 0.8.8", "mio 0.8.8",
"num_cpus", "num_cpus",
"socket2", "socket2 0.4.9",
"tokio 1.29.1", "tokio 1.32.0",
"tracing", "tracing",
] ]
@ -204,7 +204,7 @@ dependencies = [
"serde_json", "serde_json",
"serde_urlencoded 0.7.1", "serde_urlencoded 0.7.1",
"smallvec 1.11.0", "smallvec 1.11.0",
"socket2", "socket2 0.4.9",
"time 0.3.25", "time 0.3.25",
"url 2.4.0", "url 2.4.0",
] ]
@ -217,7 +217,7 @@ checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9"
dependencies = [ dependencies = [
"actix-router", "actix-router",
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 1.0.109", "syn 1.0.109",
] ]
@ -261,9 +261,9 @@ dependencies = [
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
version = "1.0.2" version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]
@ -297,9 +297,9 @@ checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
[[package]] [[package]]
name = "anyhow" name = "anyhow"
version = "1.0.72" version = "1.0.75"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
[[package]] [[package]]
name = "askama_escape" name = "askama_escape"
@ -309,13 +309,13 @@ checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
[[package]] [[package]]
name = "async-trait" name = "async-trait"
version = "0.1.72" version = "0.1.73"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
dependencies = [ dependencies = [
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 2.0.28", "syn 2.0.29",
] ]
[[package]] [[package]]
@ -386,9 +386,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "2.3.3" version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
[[package]] [[package]]
name = "block-buffer" name = "block-buffer"
@ -475,9 +475,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.0.81" version = "1.0.82"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c6b2562119bf28c3439f7f02db99faf0aa1a8cdfe5772a2ee155d32227239f0" checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01"
dependencies = [ dependencies = [
"jobserver", "jobserver",
"libc", "libc",
@ -533,18 +533,18 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.3.19" version = "4.3.23"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d" checksum = "03aef18ddf7d879c15ce20f04826ef8418101c7e528014c3eeea13321047dca3"
dependencies = [ dependencies = [
"clap_builder", "clap_builder",
] ]
[[package]] [[package]]
name = "clap_builder" name = "clap_builder"
version = "4.3.19" version = "4.3.23"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1" checksum = "f8ce6fffb678c9b80a70b6b6de0aad31df727623a70fd9a842c30cd573e2fa98"
dependencies = [ dependencies = [
"anstyle", "anstyle",
"clap_lex", "clap_lex",
@ -810,8 +810,8 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
dependencies = [ dependencies = [
"quote 1.0.32", "quote 1.0.33",
"syn 2.0.28", "syn 2.0.29",
] ]
[[package]] [[package]]
@ -828,7 +828,7 @@ checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
dependencies = [ dependencies = [
"convert_case", "convert_case",
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"rustc_version 0.4.0", "rustc_version 0.4.0",
"syn 1.0.109", "syn 1.0.109",
] ]
@ -956,7 +956,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
dependencies = [ dependencies = [
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 1.0.109", "syn 1.0.109",
"synstructure", "synstructure",
] ]
@ -980,9 +980,9 @@ checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
[[package]] [[package]]
name = "flate2" name = "flate2"
version = "1.0.26" version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010"
dependencies = [ dependencies = [
"crc32fast", "crc32fast",
"miniz_oxide", "miniz_oxide",
@ -1188,7 +1188,7 @@ dependencies = [
"http 0.2.9", "http 0.2.9",
"indexmap", "indexmap",
"slab", "slab",
"tokio 1.29.1", "tokio 1.32.0",
"tokio-util", "tokio-util",
"tracing", "tracing",
] ]
@ -1250,7 +1250,7 @@ dependencies = [
"mac", "mac",
"markup5ever 0.11.0", "markup5ever 0.11.0",
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 1.0.109", "syn 1.0.109",
] ]
@ -1313,9 +1313,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
[[package]] [[package]]
name = "httpdate" name = "httpdate"
version = "1.0.2" version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]] [[package]]
name = "humantime" name = "humantime"
@ -1370,8 +1370,8 @@ dependencies = [
"httpdate", "httpdate",
"itoa 1.0.9", "itoa 1.0.9",
"pin-project-lite", "pin-project-lite",
"socket2", "socket2 0.4.9",
"tokio 1.29.1", "tokio 1.32.0",
"tower-service", "tower-service",
"tracing", "tracing",
"want 0.3.1", "want 0.3.1",
@ -1399,7 +1399,7 @@ dependencies = [
"bytes 1.4.0", "bytes 1.4.0",
"hyper 0.14.27", "hyper 0.14.27",
"native-tls", "native-tls",
"tokio 1.29.1", "tokio 1.32.0",
"tokio-native-tls", "tokio-native-tls",
] ]
@ -1583,9 +1583,9 @@ dependencies = [
[[package]] [[package]]
name = "log" name = "log"
version = "0.4.19" version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]] [[package]]
name = "mac" name = "mac"
@ -1817,9 +1817,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]] [[package]]
name = "openssl" name = "openssl"
version = "0.10.55" version = "0.10.56"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e"
dependencies = [ dependencies = [
"bitflags 1.3.2", "bitflags 1.3.2",
"cfg-if 1.0.0", "cfg-if 1.0.0",
@ -1837,8 +1837,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [ dependencies = [
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 2.0.28", "syn 2.0.29",
] ]
[[package]] [[package]]
@ -1849,9 +1849,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]] [[package]]
name = "openssl-sys" name = "openssl-sys"
version = "0.9.90" version = "0.9.91"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac"
dependencies = [ dependencies = [
"cc", "cc",
"libc", "libc",
@ -1955,8 +1955,8 @@ dependencies = [
"pest", "pest",
"pest_meta", "pest_meta",
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 2.0.28", "syn 2.0.29",
] ]
[[package]] [[package]]
@ -2057,8 +2057,8 @@ dependencies = [
"phf_generator 0.11.2", "phf_generator 0.11.2",
"phf_shared 0.11.2", "phf_shared 0.11.2",
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 2.0.28", "syn 2.0.29",
] ]
[[package]] [[package]]
@ -2090,9 +2090,9 @@ dependencies = [
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.11" version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c516611246607d0c04186886dbb3a754368ef82c79e9827a802c6d836dd111c" checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05"
[[package]] [[package]]
name = "pin-utils" name = "pin-utils"
@ -2185,9 +2185,9 @@ dependencies = [
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.32" version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
dependencies = [ dependencies = [
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
] ]
@ -2361,16 +2361,16 @@ dependencies = [
[[package]] [[package]]
name = "redis" name = "redis"
version = "0.23.1" version = "0.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff5d95dd18a4d76650f0c2607ed8ebdbf63baf9cb934e1c233cd220c694db1d7" checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02"
dependencies = [ dependencies = [
"combine", "combine",
"itoa 1.0.9", "itoa 1.0.9",
"percent-encoding 2.3.0", "percent-encoding 2.3.0",
"ryu", "ryu",
"sha1_smol", "sha1_smol",
"socket2", "socket2 0.4.9",
"url 2.4.0", "url 2.4.0",
] ]
@ -2479,7 +2479,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"serde_urlencoded 0.7.1", "serde_urlencoded 0.7.1",
"tokio 1.29.1", "tokio 1.32.0",
"tokio-native-tls", "tokio-native-tls",
"tower-service", "tower-service",
"url 2.4.0", "url 2.4.0",
@ -2539,11 +2539,11 @@ dependencies = [
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "0.38.7" version = "0.38.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399" checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f"
dependencies = [ dependencies = [
"bitflags 2.3.3", "bitflags 2.4.0",
"errno", "errno",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys",
@ -2648,7 +2648,7 @@ version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06"
dependencies = [ dependencies = [
"bitflags 2.3.3", "bitflags 2.4.0",
"cssparser", "cssparser",
"derive_more", "derive_more",
"fxhash", "fxhash",
@ -2684,29 +2684,29 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.182" version = "1.0.183"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdb30a74471f5b7a1fa299f40b4bf1be93af61116df95465b2b5fc419331e430" checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c"
dependencies = [ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]] [[package]]
name = "serde_derive" name = "serde_derive"
version = "1.0.182" version = "1.0.183"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f4c2c6ea4bc09b5c419012eafcdb0fcef1d9119d626c8f3a0708a5b92d38a70" checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816"
dependencies = [ dependencies = [
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 2.0.28", "syn 2.0.29",
] ]
[[package]] [[package]]
name = "serde_json" name = "serde_json"
version = "1.0.104" version = "1.0.105"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360"
dependencies = [ dependencies = [
"itoa 1.0.9", "itoa 1.0.9",
"ryu", "ryu",
@ -2829,6 +2829,16 @@ dependencies = [
"winapi 0.3.9", "winapi 0.3.9",
] ]
[[package]]
name = "socket2"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877"
dependencies = [
"libc",
"windows-sys",
]
[[package]] [[package]]
name = "stable_deref_trait" name = "stable_deref_trait"
version = "1.2.0" version = "1.2.0"
@ -2882,7 +2892,7 @@ dependencies = [
"phf_generator 0.7.24", "phf_generator 0.7.24",
"phf_shared 0.7.24", "phf_shared 0.7.24",
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"string_cache_shared", "string_cache_shared",
] ]
@ -2895,7 +2905,7 @@ dependencies = [
"phf_generator 0.10.0", "phf_generator 0.10.0",
"phf_shared 0.10.0", "phf_shared 0.10.0",
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
] ]
[[package]] [[package]]
@ -2922,18 +2932,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [ dependencies = [
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"unicode-ident", "unicode-ident",
] ]
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.28" version = "2.0.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
dependencies = [ dependencies = [
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"unicode-ident", "unicode-ident",
] ]
@ -2944,16 +2954,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
dependencies = [ dependencies = [
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 1.0.109", "syn 1.0.109",
"unicode-xid 0.2.4", "unicode-xid 0.2.4",
] ]
[[package]] [[package]]
name = "tempfile" name = "tempfile"
version = "3.7.0" version = "3.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"fastrand", "fastrand",
@ -2984,22 +2994,22 @@ dependencies = [
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.44" version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f"
dependencies = [ dependencies = [
"thiserror-impl", "thiserror-impl",
] ]
[[package]] [[package]]
name = "thiserror-impl" name = "thiserror-impl"
version = "1.0.44" version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
dependencies = [ dependencies = [
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 2.0.28", "syn 2.0.29",
] ]
[[package]] [[package]]
@ -3087,11 +3097,10 @@ dependencies = [
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "1.29.1" version = "1.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9"
dependencies = [ dependencies = [
"autocfg 1.1.0",
"backtrace", "backtrace",
"bytes 1.4.0", "bytes 1.4.0",
"libc", "libc",
@ -3100,7 +3109,7 @@ dependencies = [
"parking_lot 0.12.1", "parking_lot 0.12.1",
"pin-project-lite", "pin-project-lite",
"signal-hook-registry", "signal-hook-registry",
"socket2", "socket2 0.5.3",
"tokio-macros", "tokio-macros",
"windows-sys", "windows-sys",
] ]
@ -3154,8 +3163,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [ dependencies = [
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 2.0.28", "syn 2.0.29",
] ]
[[package]] [[package]]
@ -3165,7 +3174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
dependencies = [ dependencies = [
"native-tls", "native-tls",
"tokio 1.29.1", "tokio 1.32.0",
] ]
[[package]] [[package]]
@ -3250,7 +3259,7 @@ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
"pin-project-lite", "pin-project-lite",
"tokio 1.29.1", "tokio 1.32.0",
"tracing", "tracing",
] ]
@ -3476,8 +3485,8 @@ dependencies = [
"log", "log",
"once_cell", "once_cell",
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 2.0.28", "syn 2.0.29",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -3499,7 +3508,7 @@ version = "0.2.87"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
dependencies = [ dependencies = [
"quote 1.0.32", "quote 1.0.33",
"wasm-bindgen-macro-support", "wasm-bindgen-macro-support",
] ]
@ -3510,8 +3519,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [ dependencies = [
"proc-macro2 1.0.66", "proc-macro2 1.0.66",
"quote 1.0.32", "quote 1.0.33",
"syn 2.0.28", "syn 2.0.29",
"wasm-bindgen-backend", "wasm-bindgen-backend",
"wasm-bindgen-shared", "wasm-bindgen-shared",
] ]
@ -3534,7 +3543,7 @@ dependencies = [
[[package]] [[package]]
name = "websurfx" name = "websurfx"
version = "0.16.4" version = "0.17.0"
dependencies = [ dependencies = [
"actix-cors", "actix-cors",
"actix-files", "actix-files",
@ -3556,7 +3565,7 @@ dependencies = [
"scraper", "scraper",
"serde", "serde",
"serde_json", "serde_json",
"tokio 1.29.1", "tokio 1.32.0",
] ]
[[package]] [[package]]
@ -3613,9 +3622,9 @@ dependencies = [
[[package]] [[package]]
name = "windows-targets" name = "windows-targets"
version = "0.48.1" version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
dependencies = [ dependencies = [
"windows_aarch64_gnullvm", "windows_aarch64_gnullvm",
"windows_aarch64_msvc", "windows_aarch64_msvc",
@ -3628,45 +3637,45 @@ dependencies = [
[[package]] [[package]]
name = "windows_aarch64_gnullvm" name = "windows_aarch64_gnullvm"
version = "0.48.0" version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
[[package]] [[package]]
name = "windows_aarch64_msvc" name = "windows_aarch64_msvc"
version = "0.48.0" version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
[[package]] [[package]]
name = "windows_i686_gnu" name = "windows_i686_gnu"
version = "0.48.0" version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
[[package]] [[package]]
name = "windows_i686_msvc" name = "windows_i686_msvc"
version = "0.48.0" version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
[[package]] [[package]]
name = "windows_x86_64_gnu" name = "windows_x86_64_gnu"
version = "0.48.0" version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
[[package]] [[package]]
name = "windows_x86_64_gnullvm" name = "windows_x86_64_gnullvm"
version = "0.48.0" version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
[[package]] [[package]]
name = "windows_x86_64_msvc" name = "windows_x86_64_msvc"
version = "0.48.0" version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]] [[package]]
name = "winreg" name = "winreg"

30
Cargo.toml
View File

@ -1,31 +1,31 @@
[package] [package]
name = "websurfx" name = "websurfx"
version = "0.16.4" version = "0.17.0"
edition = "2021" edition = "2021"
description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
repository = "https://github.com/neon-mmd/websurfx" repository = "https://github.com/neon-mmd/websurfx"
license = "AGPL-3.0" license = "AGPL-3.0"
[dependencies] [dependencies]
reqwest = {version="*",features=["json"]} reqwest = {version="0.11.18",features=["json"]}
tokio = {version="*",features=["full"]} tokio = {version="1.32.0",features=["full"]}
serde = {version="*",features=["derive"]} serde = {version="1.0.183",features=["derive"]}
handlebars = { version = "4.3.6", features = ["dir_source"] } handlebars = { version = "4.3.7", features = ["dir_source"] }
scraper = {version="*"} scraper = {version="0.17.1"}
actix-web = {version="4.3.1", features = ["cookies"]} actix-web = {version="4.3.1", features = ["cookies"]}
actix-files = {version="0.6.2"} actix-files = {version="0.6.2"}
actix-cors = {version="0.6.4"} actix-cors = {version="0.6.4"}
serde_json = {version="*"} serde_json = {version="1.0.105"}
fake-useragent = {version="*"} fake-useragent = {version="0.1.3"}
env_logger = {version="0.10.0"} env_logger = {version="0.10.0"}
log = {version="0.4.17"} log = {version="0.4.20"}
rlua = {version="*"} rlua = {version="0.19.7"}
redis = {version="*"} redis = {version="0.23.2"}
md5 = {version="*"} md5 = {version="0.7.0"}
rand={version="*"} rand={version="0.8.5"}
once_cell = {version="*"} once_cell = {version="1.18.0"}
error-stack = {version="0.3.1"} error-stack = {version="0.3.1"}
async-trait = {version="*"} async-trait = {version="0.1.73"}
[dev-dependencies] [dev-dependencies]
rusty-hook = "^0.11.2" rusty-hook = "^0.11.2"

2
public/templates/search.html
View File

@ -4,7 +4,7 @@
<div class="results_aggregated"> <div class="results_aggregated">
{{#if results}} {{#each results}} {{#if results}} {{#each results}}
<div class="result"> <div class="result">
<h1><a href="{{{this.visitingUrl}}}">{{{this.title}}}</a></h1> <h1><a href="{{{this.url}}}">{{{this.title}}}</a></h1>
<small>{{{this.url}}}</small> <small>{{{this.url}}}</small>
<p>{{{this.description}}}</p> <p>{{{this.description}}}</p>
<div class="upstream_engines"> <div class="upstream_engines">

2
src/bin/websurfx.rs
View File

@ -15,7 +15,7 @@ use websurfx::{config::parser::Config, run};
#[actix_web::main] #[actix_web::main]
async fn main() -> std::io::Result<()> { async fn main() -> std::io::Result<()> {
// Initialize the parsed config file. // Initialize the parsed config file.
let config = Config::parse(true).unwrap(); let config = Config::parse(false).unwrap();
log::info!( log::info!(
"started server on port {} and IP {}", "started server on port {} and IP {}",

46
src/config/parser.rs
View File

@ -34,7 +34,7 @@ pub struct Config {
pub aggregator: AggregatorConfig, pub aggregator: AggregatorConfig,
pub logging: bool, pub logging: bool,
pub debug: bool, pub debug: bool,
pub upstream_search_engines: Vec<String>, pub upstream_search_engines: Vec<crate::engines::engine_models::EngineHandler>,
pub request_timeout: u8, pub request_timeout: u8,
pub threads: u8, pub threads: u8,
} }
@ -57,7 +57,7 @@ impl Config {
/// # Arguments /// # Arguments
/// ///
/// * `logging_initialized` - It takes a boolean which ensures that the logging doesn't get /// * `logging_initialized` - It takes a boolean which ensures that the logging doesn't get
/// initialized twice. /// initialized twice. Pass false if the logger has not yet been initialized.
/// ///
/// # Error /// # Error
/// ///
@ -77,22 +77,8 @@ impl Config {
let debug: bool = globals.get::<_, bool>("debug")?; let debug: bool = globals.get::<_, bool>("debug")?;
let logging:bool= globals.get::<_, bool>("logging")?; let logging:bool= globals.get::<_, bool>("logging")?;
// Check whether logging has not been initialized before. if !logging_initialized {
if logging_initialized { set_logging_level(debug, logging);
if let Ok(pkg_env_var) = std::env::var("PKG_ENV"){
if pkg_env_var.to_lowercase() == "dev" {
env_logger::Builder::new().filter(None, LevelFilter::Trace).init();
}
} else {
// Initializing logging middleware with level set to default or info.
let mut log_level: LevelFilter = LevelFilter::Error;
if logging && debug == false {
log_level = LevelFilter::Info;
} else if debug {
log_level = LevelFilter::Debug;
};
env_logger::Builder::new().filter(None, log_level).init();
}
} }
let threads: u8 = if parsed_threads == 0 { let threads: u8 = if parsed_threads == 0 {
@ -121,12 +107,14 @@ impl Config {
.get::<_, HashMap<String, bool>>("upstream_search_engines")? .get::<_, HashMap<String, bool>>("upstream_search_engines")?
.into_iter() .into_iter()
.filter_map(|(key, value)| value.then_some(key)) .filter_map(|(key, value)| value.then_some(key))
.filter_map(|engine| crate::engines::engine_models::EngineHandler::new(&engine))
.collect(), .collect(),
request_timeout: globals.get::<_, u8>("request_timeout")?, request_timeout: globals.get::<_, u8>("request_timeout")?,
threads, threads,
}) })
}) })
} }
/// A helper function which returns an appropriate config file path checking if the config /// A helper function which returns an appropriate config file path checking if the config
/// file exists on that path. /// file exists on that path.
/// ///
@ -173,3 +161,25 @@ impl Config {
Err("Config file not found!!".to_string().into()) Err("Config file not found!!".to_string().into())
} }
} }
/// a helper function that sets the proper logging level
fn set_logging_level(debug: bool, logging: bool) {
if let Ok(pkg_env_var) = std::env::var("PKG_ENV") {
if pkg_env_var.to_lowercase() == "dev" {
env_logger::Builder::new()
.filter(None, LevelFilter::Trace)
.init();
return;
}
}
// Initializing logging middleware with level set to default or info.
let log_level = match (debug, logging) {
(true, true) => LevelFilter::Debug,
(true, false) => LevelFilter::Debug,
(false, true) => LevelFilter::Info,
(false, false) => LevelFilter::Error,
};
env_logger::Builder::new().filter(None, log_level).init();
}

8
src/engines/duckduckgo.rs
View File

@ -7,7 +7,7 @@ use std::collections::HashMap;
use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT}; use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT};
use scraper::{Html, Selector}; use scraper::{Html, Selector};
use crate::results::aggregation_models::RawSearchResult; use crate::results::aggregation_models::SearchResult;
use super::engine_models::{EngineError, SearchEngine}; use super::engine_models::{EngineError, SearchEngine};
@ -43,7 +43,7 @@ impl SearchEngine for DuckDuckGo {
page: u32, page: u32,
user_agent: String, user_agent: String,
request_timeout: u8, request_timeout: u8,
) -> Result<HashMap<String, RawSearchResult>, EngineError> { ) -> Result<HashMap<String, SearchResult>, EngineError> {
// Page number can be missing or empty string and so appropriate handling is required // Page number can be missing or empty string and so appropriate handling is required
// so that upstream server recieves valid page number. // so that upstream server recieves valid page number.
let url: String = match page { let url: String = match page {
@ -120,7 +120,7 @@ impl SearchEngine for DuckDuckGo {
Ok(document Ok(document
.select(&results) .select(&results)
.map(|result| { .map(|result| {
RawSearchResult::new( SearchResult::new(
result result
.select(&result_title) .select(&result_title)
.next() .next()
@ -147,7 +147,7 @@ impl SearchEngine for DuckDuckGo {
vec!["duckduckgo".to_string()], vec!["duckduckgo".to_string()],
) )
}) })
.map(|search_result| (search_result.visiting_url.clone(), search_result)) .map(|search_result| (search_result.url.clone(), search_result))
.collect()) .collect())
} }
} }

42
src/engines/engine_models.rs
View File

@ -1,7 +1,7 @@
//! This module provides the error enum to handle different errors associated while requesting data from //! This module provides the error enum to handle different errors associated while requesting data from
//! the upstream search engines with the search query provided by the user. //! the upstream search engines with the search query provided by the user.
use crate::results::aggregation_models::RawSearchResult; use crate::results::aggregation_models::SearchResult;
use error_stack::{IntoReport, Result, ResultExt}; use error_stack::{IntoReport, Result, ResultExt};
use std::{collections::HashMap, fmt, time::Duration}; use std::{collections::HashMap, fmt, time::Duration};
@ -43,9 +43,9 @@ impl fmt::Display for EngineError {
impl error_stack::Context for EngineError {} impl error_stack::Context for EngineError {}
/// A trait to define common behaviour for all search engines. /// A trait to define common behavior for all search engines.
#[async_trait::async_trait] #[async_trait::async_trait]
pub trait SearchEngine { pub trait SearchEngine: Sync + Send {
async fn fetch_html_from_upstream( async fn fetch_html_from_upstream(
&self, &self,
url: String, url: String,
@ -56,7 +56,7 @@ pub trait SearchEngine {
Ok(reqwest::Client::new() Ok(reqwest::Client::new()
.get(url) .get(url)
.timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server .timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
.headers(header_map) // add spoofed headers to emulate human behaviour .headers(header_map) // add spoofed headers to emulate human behavior
.send() .send()
.await .await
.into_report() .into_report()
@ -73,5 +73,37 @@ pub trait SearchEngine {
page: u32, page: u32,
user_agent: String, user_agent: String,
request_timeout: u8, request_timeout: u8,
) -> Result<HashMap<String, RawSearchResult>, EngineError>; ) -> Result<HashMap<String, SearchResult>, EngineError>;
}
pub struct EngineHandler {
engine: Box<dyn SearchEngine>,
name: &'static str,
}
impl Clone for EngineHandler {
fn clone(&self) -> Self {
Self::new(self.name).unwrap()
}
}
impl EngineHandler {
/// parses an engine name into an engine handler, returns none if the engine is unknown
pub fn new(engine_name: &str) -> Option<Self> {
let engine: (&'static str, Box<dyn SearchEngine>) =
match engine_name.to_lowercase().as_str() {
"duckduckgo" => ("duckduckgo", Box::new(super::duckduckgo::DuckDuckGo)),
"searx" => ("searx", Box::new(super::searx::Searx)),
_ => return None,
};
Some(Self {
engine: engine.1,
name: engine.0,
})
}
pub fn into_name_engine(self) -> (&'static str, Box<dyn SearchEngine>) {
(self.name, self.engine)
}
} }

8
src/engines/searx.rs
View File

@ -6,7 +6,7 @@ use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT};
use scraper::{Html, Selector}; use scraper::{Html, Selector};
use std::collections::HashMap; use std::collections::HashMap;
use crate::results::aggregation_models::RawSearchResult; use crate::results::aggregation_models::SearchResult;
use super::engine_models::{EngineError, SearchEngine}; use super::engine_models::{EngineError, SearchEngine};
use error_stack::{IntoReport, Report, Result, ResultExt}; use error_stack::{IntoReport, Report, Result, ResultExt};
@ -42,7 +42,7 @@ impl SearchEngine for Searx {
page: u32, page: u32,
user_agent: String, user_agent: String,
request_timeout: u8, request_timeout: u8,
) -> Result<HashMap<String, RawSearchResult>, EngineError> { ) -> Result<HashMap<String, SearchResult>, EngineError> {
// Page number can be missing or empty string and so appropriate handling is required // Page number can be missing or empty string and so appropriate handling is required
// so that upstream server recieves valid page number. // so that upstream server recieves valid page number.
let url: String = match page { let url: String = match page {
@ -111,7 +111,7 @@ impl SearchEngine for Searx {
Ok(document Ok(document
.select(&results) .select(&results)
.map(|result| { .map(|result| {
RawSearchResult::new( SearchResult::new(
result result
.select(&result_title) .select(&result_title)
.next() .next()
@ -137,7 +137,7 @@ impl SearchEngine for Searx {
vec!["searx".to_string()], vec!["searx".to_string()],
) )
}) })
.map(|search_result| (search_result.visiting_url.clone(), search_result)) .map(|search_result| (search_result.url.clone(), search_result))
.collect()) .collect())
} }
} }

73
src/results/aggregation_models.rs
View File

@ -5,55 +5,6 @@ use serde::{Deserialize, Serialize};
use crate::{config::parser_models::Style, engines::engine_models::EngineError}; use crate::{config::parser_models::Style, engines::engine_models::EngineError};
/// A named struct to store, serialize and deserializes the individual search result from all the
/// scraped and aggregated search results from the upstream search engines.
///
/// # Fields
///
/// * `title` - The title of the search result.
/// * `visiting_url` - The url which is accessed when clicked on it (href url in html in simple
/// words).
/// * `url` - The url to be displayed below the search result title in html.
/// * `description` - The description of the search result.
/// * `engine` - The names of the upstream engines from which this results were provided.
#[derive(Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SearchResult {
pub title: String,
pub visiting_url: String,
pub url: String,
pub description: String,
pub engine: Vec<String>,
}
impl SearchResult {
/// Constructs a new `SearchResult` with the given arguments needed for the struct.
///
/// # Arguments
///
/// * `title` - The title of the search result.
/// * `visiting_url` - The url which is accessed when clicked on it
/// (href url in html in simple words).
/// * `url` - The url to be displayed below the search result title in html.
/// * `description` - The description of the search result.
/// * `engine` - The names of the upstream engines from which this results were provided.
pub fn new(
title: String,
visiting_url: String,
url: String,
description: String,
engine: Vec<String>,
) -> Self {
SearchResult {
title,
visiting_url,
url,
description,
engine,
}
}
}
/// A named struct to store the raw scraped search results scraped search results from the /// A named struct to store the raw scraped search results scraped search results from the
/// upstream search engines before aggregating it.It derives the Clone trait which is needed /// upstream search engines before aggregating it.It derives the Clone trait which is needed
/// to write idiomatic rust using `Iterators`. /// to write idiomatic rust using `Iterators`.
@ -61,37 +12,33 @@ impl SearchResult {
/// # Fields /// # Fields
/// ///
/// * `title` - The title of the search result. /// * `title` - The title of the search result.
/// * `visiting_url` - The url which is accessed when clicked on it /// * `url` - The url which is accessed when clicked on it
/// (href url in html in simple words). /// (href url in html in simple words).
/// * `description` - The description of the search result. /// * `description` - The description of the search result.
/// * `engine` - The names of the upstream engines from which this results were provided. /// * `engine` - The names of the upstream engines from which this results were provided.
#[derive(Clone)] #[derive(Clone, Serialize, Deserialize)]
pub struct RawSearchResult { #[serde(rename_all = "camelCase")]
pub struct SearchResult {
pub title: String, pub title: String,
pub visiting_url: String, pub url: String,
pub description: String, pub description: String,
pub engine: Vec<String>, pub engine: Vec<String>,
} }
impl RawSearchResult { impl SearchResult {
/// Constructs a new `RawSearchResult` with the given arguments needed for the struct. /// Constructs a new `RawSearchResult` with the given arguments needed for the struct.
/// ///
/// # Arguments /// # Arguments
/// ///
/// * `title` - The title of the search result. /// * `title` - The title of the search result.
/// * `visiting_url` - The url which is accessed when clicked on it /// * `url` - The url which is accessed when clicked on it
/// (href url in html in simple words). /// (href url in html in simple words).
/// * `description` - The description of the search result. /// * `description` - The description of the search result.
/// * `engine` - The names of the upstream engines from which this results were provided. /// * `engine` - The names of the upstream engines from which this results were provided.
pub fn new( pub fn new(title: String, url: String, description: String, engine: Vec<String>) -> Self {
title: String, SearchResult {
visiting_url: String,
description: String,
engine: Vec<String>,
) -> Self {
RawSearchResult {
title, title,
visiting_url, url,
description, description,
engine, engine,
} }

174
src/results/aggregator.rs
View File

@ -8,18 +8,14 @@ use rand::Rng;
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use super::{ use super::{
aggregation_models::{EngineErrorInfo, RawSearchResult, SearchResult, SearchResults}, aggregation_models::{EngineErrorInfo, SearchResult, SearchResults},
user_agent::random_user_agent, user_agent::random_user_agent,
}; };
use crate::engines::{ use crate::engines::engine_models::{EngineError, EngineHandler};
duckduckgo,
engine_models::{EngineError, SearchEngine},
searx,
};
/// Aliases for long type annotations /// Aliases for long type annotations
type FutureVec = Vec<JoinHandle<Result<HashMap<String, RawSearchResult>, Report<EngineError>>>>; type FutureVec = Vec<JoinHandle<Result<HashMap<String, SearchResult>, Report<EngineError>>>>;
/// The function aggregates the scraped results from the user-selected upstream search engines. /// The function aggregates the scraped results from the user-selected upstream search engines.
/// These engines can be chosen either from the user interface (UI) or from the configuration file. /// These engines can be chosen either from the user interface (UI) or from the configuration file.
@ -64,139 +60,93 @@ pub async fn aggregate(
page: u32, page: u32,
random_delay: bool, random_delay: bool,
debug: bool, debug: bool,
upstream_search_engines: Vec<String>, upstream_search_engines: Vec<EngineHandler>,
request_timeout: u8, request_timeout: u8,
) -> Result<SearchResults, Box<dyn std::error::Error>> { ) -> Result<SearchResults, Box<dyn std::error::Error>> {
let user_agent: String = random_user_agent(); let user_agent: String = random_user_agent();
let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
// Add a random delay before making the request. // Add a random delay before making the request.
if random_delay || !debug { if random_delay || !debug {
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
let delay_secs = rng.gen_range(1..10); let delay_secs = rng.gen_range(1..10);
std::thread::sleep(Duration::from_secs(delay_secs)); tokio::time::sleep(Duration::from_secs(delay_secs)).await;
} }
// fetch results from upstream search engines simultaneously/concurrently. let mut names: Vec<&str> = vec![];
let search_engines: Vec<Box<dyn SearchEngine + Send + Sync>> = upstream_search_engines
.iter()
.map(|engine| match engine.to_lowercase().as_str() {
"duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box<dyn SearchEngine + Send + Sync>,
"searx" => Box::new(searx::Searx) as Box<dyn SearchEngine + Send + Sync>,
&_ => panic!("Config Error: Incorrect config file option provided"),
})
.collect();
let task_capacity: usize = search_engines.len(); // create tasks for upstream result fetching
let mut tasks: FutureVec = FutureVec::new();
let tasks: FutureVec = search_engines for engine_handler in upstream_search_engines {
.into_iter() let (name, search_engine) = engine_handler.into_name_engine();
.map(|search_engine| { names.push(name);
let query: String = query.clone(); let query: String = query.clone();
let user_agent: String = user_agent.clone(); let user_agent: String = user_agent.clone();
tokio::spawn(async move { tasks.push(tokio::spawn(async move {
search_engine search_engine
.results(query, page, user_agent.clone(), request_timeout) .results(query, page, user_agent.clone(), request_timeout)
.await .await
}) }));
}) }
.collect();
let mut outputs = Vec::with_capacity(task_capacity); // get upstream responses
let mut responses = Vec::with_capacity(tasks.len());
for task in tasks { for task in tasks {
if let Ok(result) = task.await { if let Ok(result) = task.await {
outputs.push(result) responses.push(result)
} }
} }
// aggregate search results, removing duplicates and handling errors the upstream engines returned
let mut result_map: HashMap<String, SearchResult> = HashMap::new();
let mut engine_errors_info: Vec<EngineErrorInfo> = Vec::new(); let mut engine_errors_info: Vec<EngineErrorInfo> = Vec::new();
// The code block `outputs.iter()` determines whether it is the first time the code is being run. let mut handle_error = |error: Report<EngineError>, engine_name: String| {
// It does this by checking the initial flag. If it is the first time, the code selects the first log::error!("Engine Error: {:?}", error);
// engine from which results are fetched and adds or extends them into the `result_map`. If the engine_errors_info.push(EngineErrorInfo::new(
// initially selected engine fails, the code automatically selects another engine to map or extend error.downcast_ref::<EngineError>().unwrap(),
// into the `result_map`. On the other hand, if an engine selected for the first time successfully engine_name.to_string(),
// fetches results and maps them into the `result_map`, the initial flag is set to false. Subsequently, ));
// the code iterates through the remaining engines one by one. It compares the fetched results from each };
// engine with the results already present in the `result_map` to identify any duplicates. If duplicate
// results are found, the code groups them together with the name of the engine from which they were for _ in 0..responses.len() {
// fetched, and automatically removes the duplicate results from the newly fetched data. let response = responses.pop().unwrap();
// let engine = names.pop().unwrap().to_string();
// Additionally, the code handles errors returned by the engines. It keeps track of which engines
// encountered errors and stores this information in a vector of structures called `EngineErrorInfo`. if result_map.is_empty() {
// Each structure in this vector contains the name of the engine and the type of error it returned. match response {
// These structures will later be added to the final `SearchResults` structure. The `SearchResults` Ok(results) => {
// structure is used to display an error box in the UI containing the relevant information from result_map = results.clone();
// the `EngineErrorInfo` structure.
//
// In summary, this code block manages the selection of engines, handling of duplicate results, and tracking
// of errors in order to populate the `result_map` and provide informative feedback to the user through the
// `SearchResults` structure.
let mut initial: bool = true;
let mut counter: usize = 0;
outputs.iter().for_each(|results| {
if initial {
match results {
Ok(result) => {
result_map.extend(result.clone());
counter += 1;
initial = false
} }
Err(error_type) => { Err(error) => {
log::error!("Engine Error: {:?}", error_type); handle_error(error, engine);
engine_errors_info.push(EngineErrorInfo::new(
error_type.downcast_ref::<EngineError>().unwrap(),
upstream_search_engines[counter].clone(),
));
counter += 1
} }
} }
} else { continue;
match results { }
Ok(result) => {
result.clone().into_iter().for_each(|(key, value)| { match response {
result_map Ok(result) => {
.entry(key) result.into_iter().for_each(|(key, value)| {
.and_modify(|result| { result_map
result.add_engines(value.clone().engine()); .entry(key)
}) .and_modify(|result| {
.or_insert_with(|| -> RawSearchResult { result.add_engines(engine.clone());
RawSearchResult::new( })
value.title.clone(), .or_insert_with(|| -> SearchResult { value });
value.visiting_url.clone(), });
value.description.clone(), }
value.engine.clone(), Err(error) => {
) handle_error(error, engine);
});
});
counter += 1
}
Err(error_type) => {
log::error!("Engine Error: {:?}", error_type);
engine_errors_info.push(EngineErrorInfo::new(
error_type.downcast_ref::<EngineError>().unwrap(),
upstream_search_engines[counter].clone(),
));
counter += 1
}
} }
} }
}); }
let results = result_map.into_values().collect();
Ok(SearchResults::new( Ok(SearchResults::new(
result_map results,
.into_iter()
.map(|(key, value)| {
SearchResult::new(
value.title,
value.visiting_url,
key,
value.description,
value.engine,
)
})
.collect(),
query.to_string(), query.to_string(),
engine_errors_info, engine_errors_info,
)) ))

57
src/server/routes.rs
View File

@ -7,12 +7,14 @@ use std::fs::read_to_string;
use crate::{ use crate::{
cache::cacher::RedisCache, cache::cacher::RedisCache,
config::parser::Config, config::parser::Config,
engines::engine_models::EngineHandler,
handler::public_paths::public_path, handler::public_paths::public_path,
results::{aggregation_models::SearchResults, aggregator::aggregate}, results::{aggregation_models::SearchResults, aggregator::aggregate},
}; };
use actix_web::{get, web, HttpRequest, HttpResponse}; use actix_web::{get, web, HttpRequest, HttpResponse};
use handlebars::Handlebars; use handlebars::Handlebars;
use serde::Deserialize; use serde::Deserialize;
use tokio::join;
/// A named struct which deserializes all the user provided search parameters and stores them. /// A named struct which deserializes all the user provided search parameters and stores them.
/// ///
@ -96,15 +98,49 @@ pub async fn search(
} }
let page = match &params.page { let page = match &params.page {
Some(page) => *page, Some(page) => *page,
None => 0, None => 1,
}; };
let url = format!( let (_, results, _) = join!(
"http://{}:{}/search?q={}&page={}", results(
config.binding_ip, config.port, query, page format!(
"http://{}:{}/search?q={}&page={}",
config.binding_ip,
config.port,
query,
page - 1
),
&config,
query.to_string(),
page - 1,
req.clone(),
),
results(
format!(
"http://{}:{}/search?q={}&page={}",
config.binding_ip, config.port, query, page
),
&config,
query.to_string(),
page,
req.clone(),
),
results(
format!(
"http://{}:{}/search?q={}&page={}",
config.binding_ip,
config.port,
query,
page + 1
),
&config,
query.to_string(),
page + 1,
req.clone(),
)
); );
let results_json = results(url, &config, query.to_string(), page, req).await?;
let page_content: String = hbs.render("search", &results_json)?; let page_content: String = hbs.render("search", &results?)?;
Ok(HttpResponse::Ok().body(page_content)) Ok(HttpResponse::Ok().body(page_content))
} }
None => Ok(HttpResponse::Found() None => Ok(HttpResponse::Found()
@ -140,12 +176,19 @@ async fn results(
{ {
Some(cookie_value) => { Some(cookie_value) => {
let cookie_value: Cookie = serde_json::from_str(cookie_value.name_value().1)?; let cookie_value: Cookie = serde_json::from_str(cookie_value.name_value().1)?;
let engines = cookie_value
.engines
.iter()
.filter_map(|name| EngineHandler::new(name))
.collect();
aggregate( aggregate(
query, query,
page, page,
config.aggregator.random_delay, config.aggregator.random_delay,
config.debug, config.debug,
cookie_value.engines, engines,
config.request_timeout, config.request_timeout,
) )
.await? .await?

4
tests/index.rs
View File

@ -8,7 +8,7 @@ fn spawn_app() -> String {
// Binding to port 0 will trigger the OS to assign a port for us. // Binding to port 0 will trigger the OS to assign a port for us.
let listener = TcpListener::bind("127.0.0.1:0").expect("Failed to bind random port"); let listener = TcpListener::bind("127.0.0.1:0").expect("Failed to bind random port");
let port = listener.local_addr().unwrap().port(); let port = listener.local_addr().unwrap().port();
let config = Config::parse(true).unwrap(); let config = Config::parse(false).unwrap();
let server = run(listener, config).expect("Failed to bind address"); let server = run(listener, config).expect("Failed to bind address");
tokio::spawn(server); tokio::spawn(server);
@ -36,7 +36,7 @@ async fn test_index() {
assert_eq!(res.status(), 200); assert_eq!(res.status(), 200);
let handlebars = handlebars(); let handlebars = handlebars();
let config = Config::parse(false).unwrap(); let config = Config::parse(true).unwrap();
let template = handlebars.render("index", &config.style).unwrap(); let template = handlebars.render("index", &config.style).unwrap();
assert_eq!(res.text().await.unwrap(), template); assert_eq!(res.text().await.unwrap(), template);
} }