mirror of
https://github.com/neon-mmd/websurfx.git
synced 2024-12-22 04:18:21 -05:00
Merge branch 'rolling' into feat-error-box-for-engine-errors
This commit is contained in:
zhou fan
GitHub
commit
575a7f95ba
@ -11,6 +11,10 @@
|
||||
],
|
||||
"language": "en",
|
||||
"noConfigSearch": true,
|
||||
"words": ["megalinter", "oxsecurity"],
|
||||
"words": [
|
||||
"megalinter",
|
||||
"oxsecurity",
|
||||
"websurfx"
|
||||
],
|
||||
"version": "0.2"
|
||||
}
|
||||
|
||||
241
Cargo.lock
generated
241
Cargo.lock
generated
@ -14,7 +14,7 @@ dependencies = [
|
||||
"futures-sink",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"tokio 1.29.1",
|
||||
"tokio 1.32.0",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
@ -90,7 +90,7 @@ dependencies = [
|
||||
"rand 0.8.5",
|
||||
"sha1",
|
||||
"smallvec 1.11.0",
|
||||
"tokio 1.29.1",
|
||||
"tokio 1.32.0",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"zstd",
|
||||
@ -102,8 +102,8 @@ version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
|
||||
dependencies = [
|
||||
"quote 1.0.32",
|
||||
"syn 2.0.28",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -126,7 +126,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"tokio 1.29.1",
|
||||
"tokio 1.32.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -142,8 +142,8 @@ dependencies = [
|
||||
"futures-util",
|
||||
"mio 0.8.8",
|
||||
"num_cpus",
|
||||
"socket2",
|
||||
"tokio 1.29.1",
|
||||
"socket2 0.4.9",
|
||||
"tokio 1.32.0",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@ -204,7 +204,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"serde_urlencoded 0.7.1",
|
||||
"smallvec 1.11.0",
|
||||
"socket2",
|
||||
"socket2 0.4.9",
|
||||
"time 0.3.25",
|
||||
"url 2.4.0",
|
||||
]
|
||||
@ -217,7 +217,7 @@ checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9"
|
||||
dependencies = [
|
||||
"actix-router",
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"quote 1.0.33",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
@ -261,9 +261,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.0.2"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
|
||||
checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
@ -297,9 +297,9 @@ checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.72"
|
||||
version = "1.0.75"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
|
||||
checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
|
||||
|
||||
[[package]]
|
||||
name = "askama_escape"
|
||||
@ -309,13 +309,13 @@ checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.72"
|
||||
version = "0.1.73"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
|
||||
checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"syn 2.0.28",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -386,9 +386,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.3.3"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
|
||||
checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
@ -475,9 +475,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.81"
|
||||
version = "1.0.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c6b2562119bf28c3439f7f02db99faf0aa1a8cdfe5772a2ee155d32227239f0"
|
||||
checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
"libc",
|
||||
@ -533,18 +533,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.3.19"
|
||||
version = "4.3.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d"
|
||||
checksum = "03aef18ddf7d879c15ce20f04826ef8418101c7e528014c3eeea13321047dca3"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.3.19"
|
||||
version = "4.3.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1"
|
||||
checksum = "f8ce6fffb678c9b80a70b6b6de0aad31df727623a70fd9a842c30cd573e2fa98"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
@ -810,8 +810,8 @@ version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
|
||||
dependencies = [
|
||||
"quote 1.0.32",
|
||||
"syn 2.0.28",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -828,7 +828,7 @@ checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"quote 1.0.33",
|
||||
"rustc_version 0.4.0",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
@ -956,7 +956,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"quote 1.0.33",
|
||||
"syn 1.0.109",
|
||||
"synstructure",
|
||||
]
|
||||
@ -980,9 +980,9 @@ checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.26"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743"
|
||||
checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
@ -1188,7 +1188,7 @@ dependencies = [
|
||||
"http 0.2.9",
|
||||
"indexmap",
|
||||
"slab",
|
||||
"tokio 1.29.1",
|
||||
"tokio 1.32.0",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
@ -1250,7 +1250,7 @@ dependencies = [
|
||||
"mac",
|
||||
"markup5ever 0.11.0",
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"quote 1.0.33",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
@ -1313,9 +1313,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
|
||||
|
||||
[[package]]
|
||||
name = "httpdate"
|
||||
version = "1.0.2"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
|
||||
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
@ -1370,8 +1370,8 @@ dependencies = [
|
||||
"httpdate",
|
||||
"itoa 1.0.9",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"tokio 1.29.1",
|
||||
"socket2 0.4.9",
|
||||
"tokio 1.32.0",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
"want 0.3.1",
|
||||
@ -1399,7 +1399,7 @@ dependencies = [
|
||||
"bytes 1.4.0",
|
||||
"hyper 0.14.27",
|
||||
"native-tls",
|
||||
"tokio 1.29.1",
|
||||
"tokio 1.32.0",
|
||||
"tokio-native-tls",
|
||||
]
|
||||
|
||||
@ -1583,9 +1583,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.19"
|
||||
version = "0.4.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
|
||||
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
|
||||
|
||||
[[package]]
|
||||
name = "mac"
|
||||
@ -1817,9 +1817,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.55"
|
||||
version = "0.10.56"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d"
|
||||
checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"cfg-if 1.0.0",
|
||||
@ -1837,8 +1837,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"syn 2.0.28",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1849,9 +1849,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
|
||||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.90"
|
||||
version = "0.9.91"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6"
|
||||
checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
@ -1955,8 +1955,8 @@ dependencies = [
|
||||
"pest",
|
||||
"pest_meta",
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"syn 2.0.28",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2057,8 +2057,8 @@ dependencies = [
|
||||
"phf_generator 0.11.2",
|
||||
"phf_shared 0.11.2",
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"syn 2.0.28",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2090,9 +2090,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.11"
|
||||
version = "0.2.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c516611246607d0c04186886dbb3a754368ef82c79e9827a802c6d836dd111c"
|
||||
checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05"
|
||||
|
||||
[[package]]
|
||||
name = "pin-utils"
|
||||
@ -2185,9 +2185,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.32"
|
||||
version = "1.0.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965"
|
||||
checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
]
|
||||
@ -2361,16 +2361,16 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "0.23.1"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff5d95dd18a4d76650f0c2607ed8ebdbf63baf9cb934e1c233cd220c694db1d7"
|
||||
checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02"
|
||||
dependencies = [
|
||||
"combine",
|
||||
"itoa 1.0.9",
|
||||
"percent-encoding 2.3.0",
|
||||
"ryu",
|
||||
"sha1_smol",
|
||||
"socket2",
|
||||
"socket2 0.4.9",
|
||||
"url 2.4.0",
|
||||
]
|
||||
|
||||
@ -2479,7 +2479,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded 0.7.1",
|
||||
"tokio 1.29.1",
|
||||
"tokio 1.32.0",
|
||||
"tokio-native-tls",
|
||||
"tower-service",
|
||||
"url 2.4.0",
|
||||
@ -2539,11 +2539,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.7"
|
||||
version = "0.38.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399"
|
||||
checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f"
|
||||
dependencies = [
|
||||
"bitflags 2.3.3",
|
||||
"bitflags 2.4.0",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
@ -2648,7 +2648,7 @@ version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06"
|
||||
dependencies = [
|
||||
"bitflags 2.3.3",
|
||||
"bitflags 2.4.0",
|
||||
"cssparser",
|
||||
"derive_more",
|
||||
"fxhash",
|
||||
@ -2684,29 +2684,29 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.182"
|
||||
version = "1.0.183"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bdb30a74471f5b7a1fa299f40b4bf1be93af61116df95465b2b5fc419331e430"
|
||||
checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.182"
|
||||
version = "1.0.183"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f4c2c6ea4bc09b5c419012eafcdb0fcef1d9119d626c8f3a0708a5b92d38a70"
|
||||
checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"syn 2.0.28",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.104"
|
||||
version = "1.0.105"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
|
||||
checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360"
|
||||
dependencies = [
|
||||
"itoa 1.0.9",
|
||||
"ryu",
|
||||
@ -2829,6 +2829,16 @@ dependencies = [
|
||||
"winapi 0.3.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.0"
|
||||
@ -2882,7 +2892,7 @@ dependencies = [
|
||||
"phf_generator 0.7.24",
|
||||
"phf_shared 0.7.24",
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"quote 1.0.33",
|
||||
"string_cache_shared",
|
||||
]
|
||||
|
||||
@ -2895,7 +2905,7 @@ dependencies = [
|
||||
"phf_generator 0.10.0",
|
||||
"phf_shared 0.10.0",
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"quote 1.0.33",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2922,18 +2932,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"quote 1.0.33",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.28"
|
||||
version = "2.0.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
|
||||
checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"quote 1.0.33",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
@ -2944,16 +2954,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"quote 1.0.33",
|
||||
"syn 1.0.109",
|
||||
"unicode-xid 0.2.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.7.0"
|
||||
version = "3.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998"
|
||||
checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"fastrand",
|
||||
@ -2984,22 +2994,22 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.44"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90"
|
||||
checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.44"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
|
||||
checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"syn 2.0.28",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3087,11 +3097,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.29.1"
|
||||
version = "1.32.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
|
||||
checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9"
|
||||
dependencies = [
|
||||
"autocfg 1.1.0",
|
||||
"backtrace",
|
||||
"bytes 1.4.0",
|
||||
"libc",
|
||||
@ -3100,7 +3109,7 @@ dependencies = [
|
||||
"parking_lot 0.12.1",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"socket2",
|
||||
"socket2 0.5.3",
|
||||
"tokio-macros",
|
||||
"windows-sys",
|
||||
]
|
||||
@ -3154,8 +3163,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"syn 2.0.28",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3165,7 +3174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
|
||||
dependencies = [
|
||||
"native-tls",
|
||||
"tokio 1.29.1",
|
||||
"tokio 1.32.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3250,7 +3259,7 @@ dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"pin-project-lite",
|
||||
"tokio 1.29.1",
|
||||
"tokio 1.32.0",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@ -3476,8 +3485,8 @@ dependencies = [
|
||||
"log",
|
||||
"once_cell",
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"syn 2.0.28",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
@ -3499,7 +3508,7 @@ version = "0.2.87"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
|
||||
dependencies = [
|
||||
"quote 1.0.32",
|
||||
"quote 1.0.33",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
@ -3510,8 +3519,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.66",
|
||||
"quote 1.0.32",
|
||||
"syn 2.0.28",
|
||||
"quote 1.0.33",
|
||||
"syn 2.0.29",
|
||||
"wasm-bindgen-backend",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
@ -3534,7 +3543,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "websurfx"
|
||||
version = "0.16.4"
|
||||
version = "0.17.0"
|
||||
dependencies = [
|
||||
"actix-cors",
|
||||
"actix-files",
|
||||
@ -3556,7 +3565,7 @@ dependencies = [
|
||||
"scraper",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio 1.29.1",
|
||||
"tokio 1.32.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -3613,9 +3622,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.48.1"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
|
||||
checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
@ -3628,45 +3637,45 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.48.0"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
|
||||
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.48.0"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
|
||||
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.48.0"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
|
||||
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.48.0"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
|
||||
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.48.0"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
|
||||
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.48.0"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
|
||||
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.48.0"
|
||||
version = "0.48.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
|
||||
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
|
||||
|
||||
[[package]]
|
||||
name = "winreg"
|
||||
|
||||
30
Cargo.toml
30
Cargo.toml
@ -1,31 +1,31 @@
|
||||
[package]
|
||||
name = "websurfx"
|
||||
version = "0.16.4"
|
||||
version = "0.17.0"
|
||||
edition = "2021"
|
||||
description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
|
||||
repository = "https://github.com/neon-mmd/websurfx"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
[dependencies]
|
||||
reqwest = {version="*",features=["json"]}
|
||||
tokio = {version="*",features=["full"]}
|
||||
serde = {version="*",features=["derive"]}
|
||||
handlebars = { version = "4.3.6", features = ["dir_source"] }
|
||||
scraper = {version="*"}
|
||||
reqwest = {version="0.11.18",features=["json"]}
|
||||
tokio = {version="1.32.0",features=["full"]}
|
||||
serde = {version="1.0.183",features=["derive"]}
|
||||
handlebars = { version = "4.3.7", features = ["dir_source"] }
|
||||
scraper = {version="0.17.1"}
|
||||
actix-web = {version="4.3.1", features = ["cookies"]}
|
||||
actix-files = {version="0.6.2"}
|
||||
actix-cors = {version="0.6.4"}
|
||||
serde_json = {version="*"}
|
||||
fake-useragent = {version="*"}
|
||||
serde_json = {version="1.0.105"}
|
||||
fake-useragent = {version="0.1.3"}
|
||||
env_logger = {version="0.10.0"}
|
||||
log = {version="0.4.17"}
|
||||
rlua = {version="*"}
|
||||
redis = {version="*"}
|
||||
md5 = {version="*"}
|
||||
rand={version="*"}
|
||||
once_cell = {version="*"}
|
||||
log = {version="0.4.20"}
|
||||
rlua = {version="0.19.7"}
|
||||
redis = {version="0.23.2"}
|
||||
md5 = {version="0.7.0"}
|
||||
rand={version="0.8.5"}
|
||||
once_cell = {version="1.18.0"}
|
||||
error-stack = {version="0.3.1"}
|
||||
async-trait = {version="*"}
|
||||
async-trait = {version="0.1.73"}
|
||||
|
||||
[dev-dependencies]
|
||||
rusty-hook = "^0.11.2"
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
<div class="results_aggregated">
|
||||
{{#if results}} {{#each results}}
|
||||
<div class="result">
|
||||
<h1><a href="{{{this.visitingUrl}}}">{{{this.title}}}</a></h1>
|
||||
<h1><a href="{{{this.url}}}">{{{this.title}}}</a></h1>
|
||||
<small>{{{this.url}}}</small>
|
||||
<p>{{{this.description}}}</p>
|
||||
<div class="upstream_engines">
|
||||
|
||||
@ -15,7 +15,7 @@ use websurfx::{config::parser::Config, run};
|
||||
#[actix_web::main]
|
||||
async fn main() -> std::io::Result<()> {
|
||||
// Initialize the parsed config file.
|
||||
let config = Config::parse(true).unwrap();
|
||||
let config = Config::parse(false).unwrap();
|
||||
|
||||
log::info!(
|
||||
"started server on port {} and IP {}",
|
||||
|
||||
@ -34,7 +34,7 @@ pub struct Config {
|
||||
pub aggregator: AggregatorConfig,
|
||||
pub logging: bool,
|
||||
pub debug: bool,
|
||||
pub upstream_search_engines: Vec<String>,
|
||||
pub upstream_search_engines: Vec<crate::engines::engine_models::EngineHandler>,
|
||||
pub request_timeout: u8,
|
||||
pub threads: u8,
|
||||
}
|
||||
@ -57,7 +57,7 @@ impl Config {
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `logging_initialized` - It takes a boolean which ensures that the logging doesn't get
|
||||
/// initialized twice.
|
||||
/// initialized twice. Pass false if the logger has not yet been initialized.
|
||||
///
|
||||
/// # Error
|
||||
///
|
||||
@ -77,22 +77,8 @@ impl Config {
|
||||
let debug: bool = globals.get::<_, bool>("debug")?;
|
||||
let logging:bool= globals.get::<_, bool>("logging")?;
|
||||
|
||||
// Check whether logging has not been initialized before.
|
||||
if logging_initialized {
|
||||
if let Ok(pkg_env_var) = std::env::var("PKG_ENV"){
|
||||
if pkg_env_var.to_lowercase() == "dev" {
|
||||
env_logger::Builder::new().filter(None, LevelFilter::Trace).init();
|
||||
}
|
||||
} else {
|
||||
// Initializing logging middleware with level set to default or info.
|
||||
let mut log_level: LevelFilter = LevelFilter::Error;
|
||||
if logging && debug == false {
|
||||
log_level = LevelFilter::Info;
|
||||
} else if debug {
|
||||
log_level = LevelFilter::Debug;
|
||||
};
|
||||
env_logger::Builder::new().filter(None, log_level).init();
|
||||
}
|
||||
if !logging_initialized {
|
||||
set_logging_level(debug, logging);
|
||||
}
|
||||
|
||||
let threads: u8 = if parsed_threads == 0 {
|
||||
@ -121,12 +107,14 @@ impl Config {
|
||||
.get::<_, HashMap<String, bool>>("upstream_search_engines")?
|
||||
.into_iter()
|
||||
.filter_map(|(key, value)| value.then_some(key))
|
||||
.filter_map(|engine| crate::engines::engine_models::EngineHandler::new(&engine))
|
||||
.collect(),
|
||||
request_timeout: globals.get::<_, u8>("request_timeout")?,
|
||||
threads,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// A helper function which returns an appropriate config file path checking if the config
|
||||
/// file exists on that path.
|
||||
///
|
||||
@ -173,3 +161,25 @@ impl Config {
|
||||
Err("Config file not found!!".to_string().into())
|
||||
}
|
||||
}
|
||||
|
||||
/// a helper function that sets the proper logging level
|
||||
fn set_logging_level(debug: bool, logging: bool) {
|
||||
if let Ok(pkg_env_var) = std::env::var("PKG_ENV") {
|
||||
if pkg_env_var.to_lowercase() == "dev" {
|
||||
env_logger::Builder::new()
|
||||
.filter(None, LevelFilter::Trace)
|
||||
.init();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Initializing logging middleware with level set to default or info.
|
||||
let log_level = match (debug, logging) {
|
||||
(true, true) => LevelFilter::Debug,
|
||||
(true, false) => LevelFilter::Debug,
|
||||
(false, true) => LevelFilter::Info,
|
||||
(false, false) => LevelFilter::Error,
|
||||
};
|
||||
|
||||
env_logger::Builder::new().filter(None, log_level).init();
|
||||
}
|
||||
|
||||
@ -7,7 +7,7 @@ use std::collections::HashMap;
|
||||
use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT};
|
||||
use scraper::{Html, Selector};
|
||||
|
||||
use crate::results::aggregation_models::RawSearchResult;
|
||||
use crate::results::aggregation_models::SearchResult;
|
||||
|
||||
use super::engine_models::{EngineError, SearchEngine};
|
||||
|
||||
@ -43,7 +43,7 @@ impl SearchEngine for DuckDuckGo {
|
||||
page: u32,
|
||||
user_agent: String,
|
||||
request_timeout: u8,
|
||||
) -> Result<HashMap<String, RawSearchResult>, EngineError> {
|
||||
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
||||
// Page number can be missing or empty string and so appropriate handling is required
|
||||
// so that upstream server recieves valid page number.
|
||||
let url: String = match page {
|
||||
@ -120,7 +120,7 @@ impl SearchEngine for DuckDuckGo {
|
||||
Ok(document
|
||||
.select(&results)
|
||||
.map(|result| {
|
||||
RawSearchResult::new(
|
||||
SearchResult::new(
|
||||
result
|
||||
.select(&result_title)
|
||||
.next()
|
||||
@ -147,7 +147,7 @@ impl SearchEngine for DuckDuckGo {
|
||||
vec!["duckduckgo".to_string()],
|
||||
)
|
||||
})
|
||||
.map(|search_result| (search_result.visiting_url.clone(), search_result))
|
||||
.map(|search_result| (search_result.url.clone(), search_result))
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
//! This module provides the error enum to handle different errors associated while requesting data from
|
||||
//! the upstream search engines with the search query provided by the user.
|
||||
|
||||
use crate::results::aggregation_models::RawSearchResult;
|
||||
use crate::results::aggregation_models::SearchResult;
|
||||
use error_stack::{IntoReport, Result, ResultExt};
|
||||
use std::{collections::HashMap, fmt, time::Duration};
|
||||
|
||||
@ -43,9 +43,9 @@ impl fmt::Display for EngineError {
|
||||
|
||||
impl error_stack::Context for EngineError {}
|
||||
|
||||
/// A trait to define common behaviour for all search engines.
|
||||
/// A trait to define common behavior for all search engines.
|
||||
#[async_trait::async_trait]
|
||||
pub trait SearchEngine {
|
||||
pub trait SearchEngine: Sync + Send {
|
||||
async fn fetch_html_from_upstream(
|
||||
&self,
|
||||
url: String,
|
||||
@ -56,7 +56,7 @@ pub trait SearchEngine {
|
||||
Ok(reqwest::Client::new()
|
||||
.get(url)
|
||||
.timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
|
||||
.headers(header_map) // add spoofed headers to emulate human behaviour
|
||||
.headers(header_map) // add spoofed headers to emulate human behavior
|
||||
.send()
|
||||
.await
|
||||
.into_report()
|
||||
@ -73,5 +73,37 @@ pub trait SearchEngine {
|
||||
page: u32,
|
||||
user_agent: String,
|
||||
request_timeout: u8,
|
||||
) -> Result<HashMap<String, RawSearchResult>, EngineError>;
|
||||
) -> Result<HashMap<String, SearchResult>, EngineError>;
|
||||
}
|
||||
|
||||
pub struct EngineHandler {
|
||||
engine: Box<dyn SearchEngine>,
|
||||
name: &'static str,
|
||||
}
|
||||
|
||||
impl Clone for EngineHandler {
|
||||
fn clone(&self) -> Self {
|
||||
Self::new(self.name).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl EngineHandler {
|
||||
/// parses an engine name into an engine handler, returns none if the engine is unknown
|
||||
pub fn new(engine_name: &str) -> Option<Self> {
|
||||
let engine: (&'static str, Box<dyn SearchEngine>) =
|
||||
match engine_name.to_lowercase().as_str() {
|
||||
"duckduckgo" => ("duckduckgo", Box::new(super::duckduckgo::DuckDuckGo)),
|
||||
"searx" => ("searx", Box::new(super::searx::Searx)),
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
Some(Self {
|
||||
engine: engine.1,
|
||||
name: engine.0,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn into_name_engine(self) -> (&'static str, Box<dyn SearchEngine>) {
|
||||
(self.name, self.engine)
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,7 +6,7 @@ use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT};
|
||||
use scraper::{Html, Selector};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::results::aggregation_models::RawSearchResult;
|
||||
use crate::results::aggregation_models::SearchResult;
|
||||
|
||||
use super::engine_models::{EngineError, SearchEngine};
|
||||
use error_stack::{IntoReport, Report, Result, ResultExt};
|
||||
@ -42,7 +42,7 @@ impl SearchEngine for Searx {
|
||||
page: u32,
|
||||
user_agent: String,
|
||||
request_timeout: u8,
|
||||
) -> Result<HashMap<String, RawSearchResult>, EngineError> {
|
||||
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
||||
// Page number can be missing or empty string and so appropriate handling is required
|
||||
// so that upstream server recieves valid page number.
|
||||
let url: String = match page {
|
||||
@ -111,7 +111,7 @@ impl SearchEngine for Searx {
|
||||
Ok(document
|
||||
.select(&results)
|
||||
.map(|result| {
|
||||
RawSearchResult::new(
|
||||
SearchResult::new(
|
||||
result
|
||||
.select(&result_title)
|
||||
.next()
|
||||
@ -137,7 +137,7 @@ impl SearchEngine for Searx {
|
||||
vec!["searx".to_string()],
|
||||
)
|
||||
})
|
||||
.map(|search_result| (search_result.visiting_url.clone(), search_result))
|
||||
.map(|search_result| (search_result.url.clone(), search_result))
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
@ -5,55 +5,6 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{config::parser_models::Style, engines::engine_models::EngineError};
|
||||
|
||||
/// A named struct to store, serialize and deserializes the individual search result from all the
|
||||
/// scraped and aggregated search results from the upstream search engines.
|
||||
///
|
||||
/// # Fields
|
||||
///
|
||||
/// * `title` - The title of the search result.
|
||||
/// * `visiting_url` - The url which is accessed when clicked on it (href url in html in simple
|
||||
/// words).
|
||||
/// * `url` - The url to be displayed below the search result title in html.
|
||||
/// * `description` - The description of the search result.
|
||||
/// * `engine` - The names of the upstream engines from which this results were provided.
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SearchResult {
|
||||
pub title: String,
|
||||
pub visiting_url: String,
|
||||
pub url: String,
|
||||
pub description: String,
|
||||
pub engine: Vec<String>,
|
||||
}
|
||||
|
||||
impl SearchResult {
|
||||
/// Constructs a new `SearchResult` with the given arguments needed for the struct.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `title` - The title of the search result.
|
||||
/// * `visiting_url` - The url which is accessed when clicked on it
|
||||
/// (href url in html in simple words).
|
||||
/// * `url` - The url to be displayed below the search result title in html.
|
||||
/// * `description` - The description of the search result.
|
||||
/// * `engine` - The names of the upstream engines from which this results were provided.
|
||||
pub fn new(
|
||||
title: String,
|
||||
visiting_url: String,
|
||||
url: String,
|
||||
description: String,
|
||||
engine: Vec<String>,
|
||||
) -> Self {
|
||||
SearchResult {
|
||||
title,
|
||||
visiting_url,
|
||||
url,
|
||||
description,
|
||||
engine,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A named struct to store the raw scraped search results scraped search results from the
|
||||
/// upstream search engines before aggregating it.It derives the Clone trait which is needed
|
||||
/// to write idiomatic rust using `Iterators`.
|
||||
@ -61,37 +12,33 @@ impl SearchResult {
|
||||
/// # Fields
|
||||
///
|
||||
/// * `title` - The title of the search result.
|
||||
/// * `visiting_url` - The url which is accessed when clicked on it
|
||||
/// * `url` - The url which is accessed when clicked on it
|
||||
/// (href url in html in simple words).
|
||||
/// * `description` - The description of the search result.
|
||||
/// * `engine` - The names of the upstream engines from which this results were provided.
|
||||
#[derive(Clone)]
|
||||
pub struct RawSearchResult {
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SearchResult {
|
||||
pub title: String,
|
||||
pub visiting_url: String,
|
||||
pub url: String,
|
||||
pub description: String,
|
||||
pub engine: Vec<String>,
|
||||
}
|
||||
|
||||
impl RawSearchResult {
|
||||
impl SearchResult {
|
||||
/// Constructs a new `RawSearchResult` with the given arguments needed for the struct.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `title` - The title of the search result.
|
||||
/// * `visiting_url` - The url which is accessed when clicked on it
|
||||
/// * `url` - The url which is accessed when clicked on it
|
||||
/// (href url in html in simple words).
|
||||
/// * `description` - The description of the search result.
|
||||
/// * `engine` - The names of the upstream engines from which this results were provided.
|
||||
pub fn new(
|
||||
title: String,
|
||||
visiting_url: String,
|
||||
description: String,
|
||||
engine: Vec<String>,
|
||||
) -> Self {
|
||||
RawSearchResult {
|
||||
pub fn new(title: String, url: String, description: String, engine: Vec<String>) -> Self {
|
||||
SearchResult {
|
||||
title,
|
||||
visiting_url,
|
||||
url,
|
||||
description,
|
||||
engine,
|
||||
}
|
||||
|
||||
@ -8,18 +8,14 @@ use rand::Rng;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use super::{
|
||||
aggregation_models::{EngineErrorInfo, RawSearchResult, SearchResult, SearchResults},
|
||||
aggregation_models::{EngineErrorInfo, SearchResult, SearchResults},
|
||||
user_agent::random_user_agent,
|
||||
};
|
||||
|
||||
use crate::engines::{
|
||||
duckduckgo,
|
||||
engine_models::{EngineError, SearchEngine},
|
||||
searx,
|
||||
};
|
||||
use crate::engines::engine_models::{EngineError, EngineHandler};
|
||||
|
||||
/// Aliases for long type annotations
|
||||
type FutureVec = Vec<JoinHandle<Result<HashMap<String, RawSearchResult>, Report<EngineError>>>>;
|
||||
type FutureVec = Vec<JoinHandle<Result<HashMap<String, SearchResult>, Report<EngineError>>>>;
|
||||
|
||||
/// The function aggregates the scraped results from the user-selected upstream search engines.
|
||||
/// These engines can be chosen either from the user interface (UI) or from the configuration file.
|
||||
@ -64,139 +60,93 @@ pub async fn aggregate(
|
||||
page: u32,
|
||||
random_delay: bool,
|
||||
debug: bool,
|
||||
upstream_search_engines: Vec<String>,
|
||||
upstream_search_engines: Vec<EngineHandler>,
|
||||
request_timeout: u8,
|
||||
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
||||
let user_agent: String = random_user_agent();
|
||||
let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
|
||||
|
||||
// Add a random delay before making the request.
|
||||
if random_delay || !debug {
|
||||
let mut rng = rand::thread_rng();
|
||||
let delay_secs = rng.gen_range(1..10);
|
||||
std::thread::sleep(Duration::from_secs(delay_secs));
|
||||
tokio::time::sleep(Duration::from_secs(delay_secs)).await;
|
||||
}
|
||||
|
||||
// fetch results from upstream search engines simultaneously/concurrently.
|
||||
let search_engines: Vec<Box<dyn SearchEngine + Send + Sync>> = upstream_search_engines
|
||||
.iter()
|
||||
.map(|engine| match engine.to_lowercase().as_str() {
|
||||
"duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box<dyn SearchEngine + Send + Sync>,
|
||||
"searx" => Box::new(searx::Searx) as Box<dyn SearchEngine + Send + Sync>,
|
||||
&_ => panic!("Config Error: Incorrect config file option provided"),
|
||||
})
|
||||
.collect();
|
||||
let mut names: Vec<&str> = vec![];
|
||||
|
||||
let task_capacity: usize = search_engines.len();
|
||||
// create tasks for upstream result fetching
|
||||
let mut tasks: FutureVec = FutureVec::new();
|
||||
|
||||
let tasks: FutureVec = search_engines
|
||||
.into_iter()
|
||||
.map(|search_engine| {
|
||||
let query: String = query.clone();
|
||||
let user_agent: String = user_agent.clone();
|
||||
tokio::spawn(async move {
|
||||
search_engine
|
||||
.results(query, page, user_agent.clone(), request_timeout)
|
||||
.await
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
for engine_handler in upstream_search_engines {
|
||||
let (name, search_engine) = engine_handler.into_name_engine();
|
||||
names.push(name);
|
||||
let query: String = query.clone();
|
||||
let user_agent: String = user_agent.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
search_engine
|
||||
.results(query, page, user_agent.clone(), request_timeout)
|
||||
.await
|
||||
}));
|
||||
}
|
||||
|
||||
let mut outputs = Vec::with_capacity(task_capacity);
|
||||
// get upstream responses
|
||||
let mut responses = Vec::with_capacity(tasks.len());
|
||||
|
||||
for task in tasks {
|
||||
if let Ok(result) = task.await {
|
||||
outputs.push(result)
|
||||
responses.push(result)
|
||||
}
|
||||
}
|
||||
|
||||
// aggregate search results, removing duplicates and handling errors the upstream engines returned
|
||||
let mut result_map: HashMap<String, SearchResult> = HashMap::new();
|
||||
let mut engine_errors_info: Vec<EngineErrorInfo> = Vec::new();
|
||||
|
||||
// The code block `outputs.iter()` determines whether it is the first time the code is being run.
|
||||
// It does this by checking the initial flag. If it is the first time, the code selects the first
|
||||
// engine from which results are fetched and adds or extends them into the `result_map`. If the
|
||||
// initially selected engine fails, the code automatically selects another engine to map or extend
|
||||
// into the `result_map`. On the other hand, if an engine selected for the first time successfully
|
||||
// fetches results and maps them into the `result_map`, the initial flag is set to false. Subsequently,
|
||||
// the code iterates through the remaining engines one by one. It compares the fetched results from each
|
||||
// engine with the results already present in the `result_map` to identify any duplicates. If duplicate
|
||||
// results are found, the code groups them together with the name of the engine from which they were
|
||||
// fetched, and automatically removes the duplicate results from the newly fetched data.
|
||||
//
|
||||
// Additionally, the code handles errors returned by the engines. It keeps track of which engines
|
||||
// encountered errors and stores this information in a vector of structures called `EngineErrorInfo`.
|
||||
// Each structure in this vector contains the name of the engine and the type of error it returned.
|
||||
// These structures will later be added to the final `SearchResults` structure. The `SearchResults`
|
||||
// structure is used to display an error box in the UI containing the relevant information from
|
||||
// the `EngineErrorInfo` structure.
|
||||
//
|
||||
// In summary, this code block manages the selection of engines, handling of duplicate results, and tracking
|
||||
// of errors in order to populate the `result_map` and provide informative feedback to the user through the
|
||||
// `SearchResults` structure.
|
||||
let mut initial: bool = true;
|
||||
let mut counter: usize = 0;
|
||||
outputs.iter().for_each(|results| {
|
||||
if initial {
|
||||
match results {
|
||||
Ok(result) => {
|
||||
result_map.extend(result.clone());
|
||||
counter += 1;
|
||||
initial = false
|
||||
let mut handle_error = |error: Report<EngineError>, engine_name: String| {
|
||||
log::error!("Engine Error: {:?}", error);
|
||||
engine_errors_info.push(EngineErrorInfo::new(
|
||||
error.downcast_ref::<EngineError>().unwrap(),
|
||||
engine_name.to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
for _ in 0..responses.len() {
|
||||
let response = responses.pop().unwrap();
|
||||
let engine = names.pop().unwrap().to_string();
|
||||
|
||||
if result_map.is_empty() {
|
||||
match response {
|
||||
Ok(results) => {
|
||||
result_map = results.clone();
|
||||
}
|
||||
Err(error_type) => {
|
||||
log::error!("Engine Error: {:?}", error_type);
|
||||
engine_errors_info.push(EngineErrorInfo::new(
|
||||
error_type.downcast_ref::<EngineError>().unwrap(),
|
||||
upstream_search_engines[counter].clone(),
|
||||
));
|
||||
counter += 1
|
||||
Err(error) => {
|
||||
handle_error(error, engine);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
match results {
|
||||
Ok(result) => {
|
||||
result.clone().into_iter().for_each(|(key, value)| {
|
||||
result_map
|
||||
.entry(key)
|
||||
.and_modify(|result| {
|
||||
result.add_engines(value.clone().engine());
|
||||
})
|
||||
.or_insert_with(|| -> RawSearchResult {
|
||||
RawSearchResult::new(
|
||||
value.title.clone(),
|
||||
value.visiting_url.clone(),
|
||||
value.description.clone(),
|
||||
value.engine.clone(),
|
||||
)
|
||||
});
|
||||
});
|
||||
counter += 1
|
||||
}
|
||||
Err(error_type) => {
|
||||
log::error!("Engine Error: {:?}", error_type);
|
||||
engine_errors_info.push(EngineErrorInfo::new(
|
||||
error_type.downcast_ref::<EngineError>().unwrap(),
|
||||
upstream_search_engines[counter].clone(),
|
||||
));
|
||||
counter += 1
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
match response {
|
||||
Ok(result) => {
|
||||
result.into_iter().for_each(|(key, value)| {
|
||||
result_map
|
||||
.entry(key)
|
||||
.and_modify(|result| {
|
||||
result.add_engines(engine.clone());
|
||||
})
|
||||
.or_insert_with(|| -> SearchResult { value });
|
||||
});
|
||||
}
|
||||
Err(error) => {
|
||||
handle_error(error, engine);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = result_map.into_values().collect();
|
||||
|
||||
Ok(SearchResults::new(
|
||||
result_map
|
||||
.into_iter()
|
||||
.map(|(key, value)| {
|
||||
SearchResult::new(
|
||||
value.title,
|
||||
value.visiting_url,
|
||||
key,
|
||||
value.description,
|
||||
value.engine,
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
results,
|
||||
query.to_string(),
|
||||
engine_errors_info,
|
||||
))
|
||||
|
||||
@ -7,12 +7,14 @@ use std::fs::read_to_string;
|
||||
use crate::{
|
||||
cache::cacher::RedisCache,
|
||||
config::parser::Config,
|
||||
engines::engine_models::EngineHandler,
|
||||
handler::public_paths::public_path,
|
||||
results::{aggregation_models::SearchResults, aggregator::aggregate},
|
||||
};
|
||||
use actix_web::{get, web, HttpRequest, HttpResponse};
|
||||
use handlebars::Handlebars;
|
||||
use serde::Deserialize;
|
||||
use tokio::join;
|
||||
|
||||
/// A named struct which deserializes all the user provided search parameters and stores them.
|
||||
///
|
||||
@ -96,15 +98,49 @@ pub async fn search(
|
||||
}
|
||||
let page = match ¶ms.page {
|
||||
Some(page) => *page,
|
||||
None => 0,
|
||||
None => 1,
|
||||
};
|
||||
|
||||
let url = format!(
|
||||
"http://{}:{}/search?q={}&page={}",
|
||||
config.binding_ip, config.port, query, page
|
||||
let (_, results, _) = join!(
|
||||
results(
|
||||
format!(
|
||||
"http://{}:{}/search?q={}&page={}",
|
||||
config.binding_ip,
|
||||
config.port,
|
||||
query,
|
||||
page - 1
|
||||
),
|
||||
&config,
|
||||
query.to_string(),
|
||||
page - 1,
|
||||
req.clone(),
|
||||
),
|
||||
results(
|
||||
format!(
|
||||
"http://{}:{}/search?q={}&page={}",
|
||||
config.binding_ip, config.port, query, page
|
||||
),
|
||||
&config,
|
||||
query.to_string(),
|
||||
page,
|
||||
req.clone(),
|
||||
),
|
||||
results(
|
||||
format!(
|
||||
"http://{}:{}/search?q={}&page={}",
|
||||
config.binding_ip,
|
||||
config.port,
|
||||
query,
|
||||
page + 1
|
||||
),
|
||||
&config,
|
||||
query.to_string(),
|
||||
page + 1,
|
||||
req.clone(),
|
||||
)
|
||||
);
|
||||
let results_json = results(url, &config, query.to_string(), page, req).await?;
|
||||
let page_content: String = hbs.render("search", &results_json)?;
|
||||
|
||||
let page_content: String = hbs.render("search", &results?)?;
|
||||
Ok(HttpResponse::Ok().body(page_content))
|
||||
}
|
||||
None => Ok(HttpResponse::Found()
|
||||
@ -140,12 +176,19 @@ async fn results(
|
||||
{
|
||||
Some(cookie_value) => {
|
||||
let cookie_value: Cookie = serde_json::from_str(cookie_value.name_value().1)?;
|
||||
|
||||
let engines = cookie_value
|
||||
.engines
|
||||
.iter()
|
||||
.filter_map(|name| EngineHandler::new(name))
|
||||
.collect();
|
||||
|
||||
aggregate(
|
||||
query,
|
||||
page,
|
||||
config.aggregator.random_delay,
|
||||
config.debug,
|
||||
cookie_value.engines,
|
||||
engines,
|
||||
config.request_timeout,
|
||||
)
|
||||
.await?
|
||||
|
||||
@ -8,7 +8,7 @@ fn spawn_app() -> String {
|
||||
// Binding to port 0 will trigger the OS to assign a port for us.
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("Failed to bind random port");
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
let config = Config::parse(true).unwrap();
|
||||
let config = Config::parse(false).unwrap();
|
||||
let server = run(listener, config).expect("Failed to bind address");
|
||||
|
||||
tokio::spawn(server);
|
||||
@ -36,7 +36,7 @@ async fn test_index() {
|
||||
assert_eq!(res.status(), 200);
|
||||
|
||||
let handlebars = handlebars();
|
||||
let config = Config::parse(false).unwrap();
|
||||
let config = Config::parse(true).unwrap();
|
||||
let template = handlebars.render("index", &config.style).unwrap();
|
||||
assert_eq!(res.text().await.unwrap(), template);
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user