alamin/websurfx
alamin/websurfx
0
0
Fork 0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-21 21:48:21 -05:00
Code Issues Releases Activity

✨ feat: add condition to filter results only when safe_search level is

Browse Source 
set to 3 or above (#201)
This commit is contained in:
neon_arch 2023-09-02 17:38:46 +03:00
parent 8eeaf19cbd
commit d6463f0872
1 changed files with 23 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
src/results/aggregator.rs
View File

@ -70,6 +70,7 @@ pub async fn aggregate(
debug: bool, debug: bool,
upstream_search_engines: Vec<EngineHandler>, upstream_search_engines: Vec<EngineHandler>,
request_timeout: u8, request_timeout: u8,
safe_search: u8,
) -> Result<SearchResults, Box<dyn std::error::Error>> { ) -> Result<SearchResults, Box<dyn std::error::Error>> {
let user_agent: String = random_user_agent(); let user_agent: String = random_user_agent();
@ -92,7 +93,13 @@ pub async fn aggregate(
let user_agent: String = user_agent.clone(); let user_agent: String = user_agent.clone();
tasks.push(tokio::spawn(async move { tasks.push(tokio::spawn(async move {
search_engine search_engine
.results(query, page, user_agent.clone(), request_timeout) .results(
query,
page,
user_agent.clone(),
request_timeout,
safe_search,
)
.await .await
})); }));
} }
@ -151,20 +158,22 @@ pub async fn aggregate(
} }
} }
let mut blacklist_map: HashMap<String, SearchResult> = HashMap::new(); if safe_search >= 3 {
filter_with_lists( let mut blacklist_map: HashMap<String, SearchResult> = HashMap::new();
&mut result_map, filter_with_lists(
&mut blacklist_map, &mut result_map,
&file_path(FileType::BlockList)?, &mut blacklist_map,
)?; &file_path(FileType::BlockList)?,
)?;
filter_with_lists( filter_with_lists(
&mut blacklist_map, &mut blacklist_map,
&mut result_map, &mut result_map,
&file_path(FileType::AllowList)?, &file_path(FileType::AllowList)?,
)?; )?;
drop(blacklist_map); drop(blacklist_map);
}
let results: Vec<SearchResult> = result_map.into_values().collect(); let results: Vec<SearchResult> = result_map.into_values().collect();
@ -194,7 +203,7 @@ pub fn filter_with_lists(
let mut reader = BufReader::new(File::open(file_path)?); let mut reader = BufReader::new(File::open(file_path)?);
for line in reader.by_ref().lines() { for line in reader.by_ref().lines() {
let re = Regex::new(&line?)?; let re = Regex::new(line?.trim())?;
// Iterate over each search result in the map and check if it matches the regex pattern // Iterate over each search result in the map and check if it matches the regex pattern
for (url, search_result) in map_to_be_filtered.clone().into_iter() { for (url, search_result) in map_to_be_filtered.clone().into_iter() {