diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index fe6c370..254ab16 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; -use reqwest::header::{HeaderMap, CONTENT_TYPE, REFERER, USER_AGENT}; +use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT}; use scraper::{Html, Selector}; use crate::search_results_handler::aggregation_models::RawSearchResult; @@ -50,7 +50,8 @@ pub async fn results( let mut header_map = HeaderMap::new(); header_map.insert(USER_AGENT, user_agent.parse()?); header_map.insert(REFERER, "https://google.com/".parse()?); - header_map.insert(CONTENT_TYPE, "text/html; charset=UTF-8".parse()?); + header_map.insert(CONTENT_TYPE, "application/x-www-form-urlencoded".parse()?); + header_map.insert(COOKIE, "kl=wt-wt".parse()?); // fetch the html from upstream duckduckgo engine // TODO: Write better error handling code to handle no results case. diff --git a/src/engines/searx.rs b/src/engines/searx.rs index afb34cb..8812dd9 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -2,7 +2,7 @@ //! by querying the upstream searx search engine instance with user provided query and with a page //! number if provided. -use reqwest::header::{HeaderMap, CONTENT_TYPE, REFERER, USER_AGENT}; +use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT}; use scraper::{Html, Selector}; use std::collections::HashMap; @@ -38,6 +38,7 @@ pub async fn results( header_map.insert(USER_AGENT, user_agent.parse()?); header_map.insert(REFERER, "https://google.com/".parse()?); header_map.insert(CONTENT_TYPE, "application/x-www-form-urlencoded".parse()?); + header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse()?); // fetch the html from upstream searx instance engine // TODO: Write better error handling code to handle no results case.