Merge 169371bf29 into 669e365913

feat: add new helper function to fetch upstream search engine JSON response (#504 )
Co-authored-by: neon_arch <mustafadhuleb53@gmail.com>
2024-11-24 23:18:22 -05:00 · 2024-01-30 13:38:27 +00:00 · 2024-01-30 13:37:50 +00:00 · 2024-01-27 22:30:59 +05:30 · 2024-01-27 21:48:15 +05:30 · 2024-01-27 21:47:47 +05:30
4 changed files with 44 additions and 6 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -4146,7 +4146,7 @@ checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10"

 [[package]]
 name = "websurfx"
-version = "1.9.4"
+version = "1.9.6"
 dependencies = [
 "actix-cors",
 "actix-files",
--- a/Cargo.toml
+++ b/Cargo.toml
@ -1,6 +1,6 @@
 [package]
 name = "websurfx"
-version = "1.9.4"
+version = "1.9.6"
 edition = "2021"
 description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
 repository = "https://github.com/neon-mmd/websurfx"
--- a/docs/instances.md
+++ b/docs/instances.md
@ -4,10 +4,12 @@

 This page provides a list of `Websurfx` instances provided by us and our community. 

-|URL|Network|Version|Location|Behind Cloudflare?|Maintained By|TLS|IPv6|Comment|
+|URL|Network|Version|Location|Behind Cloudflare?|Status|TLS|IPv6|Comment|
 |-|-|-|-|-|-|-|-|-|
-|https://websurfx.co/|www|edge|🇺🇸 US|||✅|❌||
-|https://websurfx.onrender.com/|www|edge|🇺🇸 US|||✅|❌||
-|https://alamin655-websurfx.hf.space/|www|v0.21.4|🇺🇸 US||[websurfx project](https://github.com/neon-mmd/websurfx)|✅|❌||
+|https://websurfx.co|www|rolling|🇺🇸 US||<a href="https://status.websurfx.pp.ua"><img src="https://status.websurfx.pp.ua/api/badge/2/status"></a>|✅|❌||
+|https://alamin655-spacex.hf.space|www|rolling|🇺🇸 US||<a href="https://status.websurfx.pp.ua"><img src="https://status.websurfx.pp.ua/api/badge/5/status"></a>|✅|❌||
+|https://websurfx.instance.pp.ua|www|rolling|🇺🇸 US||<a href="https://status.websurfx.pp.ua"><img src="https://status.websurfx.pp.ua/api/badge/7/status"></a>|✅|✅||
+|https://alamin655-websurfx.hf.space|www|stable|🇺🇸 US||<a href="https://status.websurfx.pp.ua"><img src="https://status.websurfx.pp.ua/api/badge/6/status"></a>|✅|❌||
+

 [⬅️ Go back to Home](./README.md)
--- a/src/models/engine_models.rs
+++ b/src/models/engine_models.rs
@ -86,6 +86,42 @@ pub trait SearchEngine: Sync + Send {
            .change_context(EngineError::RequestError)?)
    }

+    /// This helper function fetches/requests the json search results from the upstream search engine as a vector of bytes.
+    ///
+    /// # Arguments
+    ///
+    /// * `url` - It takes the url of the upstream search engine with the user requested search
+    /// query appended in the search parameters.
+    /// * `header_map` - It takes the http request headers to be sent to the upstream engine in
+    /// order to prevent being detected as a bot. It takes the header as a HeaderMap type.
+    /// * `request_timeout` - It takes the request timeout value as seconds which is used to limit
+    /// the amount of time for each request to remain connected when until the results can be provided
+    /// by the upstream engine.
+    ///
+    /// # Error
+    ///
+    /// It returns the html data as a vector of bytes if the upstream engine provides the data as expected
+    /// otherwise it returns a custom `EngineError`.
+    async fn fetch_json_as_bytes_from_upstream(
+        &self,
+        url: &str,
+        header_map: reqwest::header::HeaderMap,
+        client: &Client,
+    ) -> Result<Vec<u8>, EngineError> {
+        // fetch the json response from upstream search engine
+
+        Ok(client
+            .get(url)
+            .headers(header_map) // add spoofed headers to emulate human behavior
+            .send()
+            .await
+            .change_context(EngineError::RequestError)?
+            .bytes()
+            .await
+            .change_context(EngineError::RequestError)?
+            .to_vec())
+    }
+
    /// This function scrapes results from the upstream engine and puts all the scraped results like
    /// title, visiting_url (href in html),engine (from which engine it was fetched from) and description
    /// in a RawSearchResult and then adds that to HashMap whose keys are url and values are RawSearchResult
Author	SHA1	Message	Date
alamin655	341b972412	Merge `169371bf29` into `669e365913`	2024-01-30 13:38:27 +00:00
Jann Marc Villablanca	669e365913	feat: add new helper function to fetch upstream search engine JSON response (#504 ) Co-authored-by: neon_arch <mustafadhuleb53@gmail.com>	2024-01-30 13:37:50 +00:00
alamin655	169371bf29	Remove unnecessary line Co-authored-by: neon_arch <mustafadhuleb53@gmail.com>	2024-01-27 22:30:59 +05:30
alamin655	40d4d84bdc	Merge branch 'rolling' into instance	2024-01-27 21:48:15 +05:30
alamin655	af1b65ac5f	🔖 chore(release): bump the app version (#495 )	2024-01-27 21:47:47 +05:30
alamin655	a22eb724b3	🔖 chore(release): bump the app version (#495 )	2024-01-27 21:47:17 +05:30
alamin655	227b073251	🔖 chore(release): bump the app version (#495 )	2024-01-27 20:37:21 +05:30
alamin655	d715833250	🔖 chore(release): bump the app version (#495 )	2024-01-27 20:37:10 +05:30
alamin655	a3f5880c82	add 2 additional instance Added two new instance - https://websurfx.instance.pp.ua - https://alamin655-spacex.hf.space Changed the `Managed by` column to `Status`	2024-01-27 20:27:22 +05:30