alamin/websurfx
alamin/websurfx
0
0
Fork 0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-10-18 06:22:53 -04:00
Code Issues Releases Activity

Merge pull request #469 from neon-mmd/FIX/468_pagination-for-the-upstream-search-engines-not-working

Browse Source 
🚑️ Pagination code for the upstream search engines
This commit is contained in:
alamin655 2023-12-30 21:37:06 +05:30 committed by GitHub
commit ddb10f6584
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 32 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Cargo.lock generated
View File

@ -79,7 +79,7 @@ dependencies = [
"actix-rt", "actix-rt",
"actix-service", "actix-service",
"actix-utils", "actix-utils",
"ahash 0.8.6", "ahash 0.8.7",
"base64 0.21.5", "base64 0.21.5",
"bitflags 2.4.1", "bitflags 2.4.1",
"bytes 1.5.0", "bytes 1.5.0",
@ -190,7 +190,7 @@ dependencies = [
"actix-service", "actix-service",
"actix-utils", "actix-utils",
"actix-web-codegen", "actix-web-codegen",
"ahash 0.8.6", "ahash 0.8.7",
"bytes 1.5.0", "bytes 1.5.0",
"bytestring", "bytestring",
"cfg-if 1.0.0", "cfg-if 1.0.0",
@ -255,9 +255,9 @@ dependencies = [
[[package]] [[package]]
name = "ahash" name = "ahash"
version = "0.8.6" version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"getrandom", "getrandom",
@ -489,9 +489,9 @@ dependencies = [
[[package]] [[package]]
name = "bstr" name = "bstr"
version = "1.8.0" version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc"
dependencies = [ dependencies = [
"memchr", "memchr",
"serde", "serde",
@ -637,18 +637,18 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.4.11" version = "4.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d"
dependencies = [ dependencies = [
"clap_builder", "clap_builder",
] ]
[[package]] [[package]]
name = "clap_builder" name = "clap_builder"
version = "4.4.11" version = "4.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9"
dependencies = [ dependencies = [
"anstyle", "anstyle",
"clap_lex", "clap_lex",
@ -1459,7 +1459,7 @@ version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
dependencies = [ dependencies = [
"ahash 0.8.6", "ahash 0.8.7",
"bumpalo", "bumpalo",
] ]
@ -1973,9 +1973,9 @@ checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.6.4" version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
[[package]] [[package]]
name = "memoffset" name = "memoffset"
@ -3066,7 +3066,7 @@ version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "585480e3719b311b78a573db1c9d9c4c1f8010c2dee4cc59c2efe58ea4dbc3e1" checksum = "585480e3719b311b78a573db1c9d9c4c1f8010c2dee4cc59c2efe58ea4dbc3e1"
dependencies = [ dependencies = [
"ahash 0.8.6", "ahash 0.8.7",
"cssparser 0.31.2", "cssparser 0.31.2",
"ego-tree", "ego-tree",
"html5ever 0.26.0", "html5ever 0.26.0",
@ -4056,7 +4056,7 @@ checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10"
[[package]] [[package]]
name = "websurfx" name = "websurfx"
version = "1.6.11" version = "1.7.3"
dependencies = [ dependencies = [
"actix-cors", "actix-cors",
"actix-files", "actix-files",

2
Cargo.toml
View File

@ -1,6 +1,6 @@
[package] [package]
name = "websurfx" name = "websurfx"
version = "1.6.11" version = "1.7.3"
edition = "2021" edition = "2021"
description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
repository = "https://github.com/neon-mmd/websurfx" repository = "https://github.com/neon-mmd/websurfx"

9
src/engines/duckduckgo.rs
View File

@ -51,15 +51,14 @@ impl SearchEngine for DuckDuckGo {
// Page number can be missing or empty string and so appropriate handling is required // Page number can be missing or empty string and so appropriate handling is required
// so that upstream server recieves valid page number. // so that upstream server recieves valid page number.
let url: String = match page { let url: String = match page {
1 | 0 => { 0 => {
format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js") format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js")
} }
_ => { _ => {
format!( format!(
"https://duckduckgo.com/html/?q={}&s={}&dc={}&v=1&o=json&api=/d.js", "https://duckduckgo.com/html/?q={query}&s={}&dc={}&v=1&o=json&api=/d.js",
query, page * 30,
(page / 2 + (page % 2)) * 30, page * 30 + 1
(page / 2 + (page % 2)) * 30 + 1
) )
} }
}; };

13
src/engines/librex.rs
View File

@ -65,17 +65,10 @@ impl SearchEngine for LibreX {
) -> Result<HashMap<String, SearchResult>, EngineError> { ) -> Result<HashMap<String, SearchResult>, EngineError> {
// Page number can be missing or empty string and so appropriate handling is required // Page number can be missing or empty string and so appropriate handling is required
// so that upstream server recieves valid page number. // so that upstream server recieves valid page number.
let url: String = match page { let url: String = format!(
1 | 0 => {
format!("https://search.ahwx.org/search.php?q={query}&p=0&t=10")
}
_ => {
format!(
"https://search.ahwx.org/search.php?q={query}&p={}&t=10", "https://search.ahwx.org/search.php?q={query}&p={}&t=10",
page * 10, page * 10
) );
}
};
// initializing HeaderMap and adding appropriate headers. // initializing HeaderMap and adding appropriate headers.
let header_map = HeaderMap::try_from(&HashMap::from([ let header_map = HeaderMap::try_from(&HashMap::from([

12
src/engines/searx.rs
View File

@ -50,14 +50,10 @@ impl SearchEngine for Searx {
safe_search = 2; safe_search = 2;
}; };
let url: String = match page { let url: String = format!(
0 | 1 => { "https://searx.be/search?q={query}&pageno={}&safesearch={safe_search}",
format!("https://searx.be/search?q={query}&pageno=1&safesearch={safe_search}") page + 1
} );
_ => {
format!("https://searx.be/search?q={query}&pageno={page}&safesearch={safe_search}")
}
};
// initializing headers and adding appropriate headers. // initializing headers and adding appropriate headers.
let header_map = HeaderMap::try_from(&HashMap::from([ let header_map = HeaderMap::try_from(&HashMap::from([

11
src/engines/startpage.rs
View File

@ -50,17 +50,10 @@ impl SearchEngine for Startpage {
) -> Result<HashMap<String, SearchResult>, EngineError> { ) -> Result<HashMap<String, SearchResult>, EngineError> {
// Page number can be missing or empty string and so appropriate handling is required // Page number can be missing or empty string and so appropriate handling is required
// so that upstream server recieves valid page number. // so that upstream server recieves valid page number.
let url: String = match page { let url: String = format!(
1 | 0 => {
format!("https://startpage.com/do/dsearch?q={query}&num=10&start=0")
}
_ => {
format!(
"https://startpage.com/do/dsearch?q={query}&num=10&start={}", "https://startpage.com/do/dsearch?q={query}&num=10&start={}",
page * 10, page * 10,
) );
}
};
// initializing HeaderMap and adding appropriate headers. // initializing HeaderMap and adding appropriate headers.
let header_map = HeaderMap::try_from(&HashMap::from([ let header_map = HeaderMap::try_from(&HashMap::from([