From fbf73634ee72059f59acb7611e07d1830247c332 Mon Sep 17 00:00:00 2001 From: abdulahad5112 <123822052+abdulahad5112@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:39:33 +0530 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=90=9B=20Undeclared=20mini-mocha=20cr?= =?UTF-8?q?ate=20error=20when=20building=20the=20app=20with=20features=20o?= =?UTF-8?q?ther=20than=20memory-cache=20#493=20(#501)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cache/cacher.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cache/cacher.rs b/src/cache/cacher.rs index f323395..53d8050 100644 --- a/src/cache/cacher.rs +++ b/src/cache/cacher.rs @@ -4,6 +4,7 @@ use error_stack::Report; #[cfg(feature = "memory-cache")] use mini_moka::sync::Cache as MokaCache; +#[cfg(feature = "memory-cache")] use mini_moka::sync::ConcurrentCacheExt; #[cfg(feature = "memory-cache")] From 851ea314a7b9e85fa718065dd5fd0c74ddfefa55 Mon Sep 17 00:00:00 2001 From: abdulahad5112 <123822052+abdulahad5112@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:44:31 +0530 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=90=9B=20parsed=5Fcet=20not=20found?= =?UTF-8?q?=20in=20scope=20error=20when=20building=20the=20app=20with=20th?= =?UTF-8?q?e=20no-cache=20feature=20#498=20(#502)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: alamin655 <129589283+alamin655@users.noreply.github.com> --- src/config/parser.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config/parser.rs b/src/config/parser.rs index 20a4a1a..63329c5 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -98,6 +98,7 @@ impl Config { #[cfg(any(feature = "redis-cache", feature = "memory-cache"))] let parsed_cet = globals.get::<_, u16>("cache_expiry_time")?; + #[cfg(any(feature = "redis-cache", feature = "memory-cache"))] let cache_expiry_time = match parsed_cet { 0..=59 => { log::error!( From b2cbc5eaa5dadd74e17840cbae64bb84b9feaf79 Mon Sep 17 00:00:00 2001 From: abdulahad5112 <123822052+abdulahad5112@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:50:14 +0530 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=93=9D=20Maintained=20badge/shield=20?= =?UTF-8?q?status=20in=20the=20readme=20from=20stale=20to=20yes/maintained?= =?UTF-8?q?=20#500=20(#503)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: alamin655 <129589283+alamin655@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a7377d..0aeb23e 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Maintenance From 669e36591326d95aa431fe6693b3dbeead0a639a Mon Sep 17 00:00:00 2001 From: Jann Marc Villablanca <31008330+jfvillablanca@users.noreply.github.com> Date: Tue, 30 Jan 2024 21:37:50 +0800 Subject: [PATCH 4/4] feat: add new helper function to fetch upstream search engine JSON response (#504) Co-authored-by: neon_arch --- src/models/engine_models.rs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/models/engine_models.rs b/src/models/engine_models.rs index 3108e6e..4d56836 100644 --- a/src/models/engine_models.rs +++ b/src/models/engine_models.rs @@ -86,6 +86,42 @@ pub trait SearchEngine: Sync + Send { .change_context(EngineError::RequestError)?) } + /// This helper function fetches/requests the json search results from the upstream search engine as a vector of bytes. + /// + /// # Arguments + /// + /// * `url` - It takes the url of the upstream search engine with the user requested search + /// query appended in the search parameters. + /// * `header_map` - It takes the http request headers to be sent to the upstream engine in + /// order to prevent being detected as a bot. It takes the header as a HeaderMap type. + /// * `request_timeout` - It takes the request timeout value as seconds which is used to limit + /// the amount of time for each request to remain connected when until the results can be provided + /// by the upstream engine. + /// + /// # Error + /// + /// It returns the html data as a vector of bytes if the upstream engine provides the data as expected + /// otherwise it returns a custom `EngineError`. + async fn fetch_json_as_bytes_from_upstream( + &self, + url: &str, + header_map: reqwest::header::HeaderMap, + client: &Client, + ) -> Result, EngineError> { + // fetch the json response from upstream search engine + + Ok(client + .get(url) + .headers(header_map) // add spoofed headers to emulate human behavior + .send() + .await + .change_context(EngineError::RequestError)? + .bytes() + .await + .change_context(EngineError::RequestError)? + .to_vec()) + } + /// This function scrapes results from the upstream engine and puts all the scraped results like /// title, visiting_url (href in html),engine (from which engine it was fetched from) and description /// in a RawSearchResult and then adds that to HashMap whose keys are url and values are RawSearchResult