mirror of
https://github.com/neon-mmd/websurfx.git
synced 2024-12-22 20:38:22 -05:00
Merge branch 'rolling' into ability-to-have-config-on-diff-paths
This commit is contained in:
commit
e6bb981133
11
.github/workflows/rust_format.yml
vendored
11
.github/workflows/rust_format.yml
vendored
@ -19,7 +19,16 @@ jobs:
|
||||
profile: minimal
|
||||
toolchain: stable
|
||||
components: rustfmt, clippy
|
||||
|
||||
- name: Format
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: fmt
|
||||
args: -- --check
|
||||
- name: Clippy
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
command: clippy
|
||||
args: --all-features --all-targets --all
|
||||
- name: Run cargo check
|
||||
uses: actions-rs/cargo@v1
|
||||
with:
|
||||
|
BIN
public/images/websurfx_logo.png
Normal file
BIN
public/images/websurfx_logo.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 8.1 KiB |
@ -1,6 +1,6 @@
|
||||
{{>header this}}
|
||||
<main class="search-container">
|
||||
<img src="images/fps_logo.png" alt="Websurfx meta-search engine logo" />
|
||||
<img src="../images/websurfx_logo.png" alt="Websurfx meta-search engine logo" />
|
||||
{{>search_bar}}
|
||||
</main>
|
||||
<script src="static/index.js"></script>
|
||||
|
2
src/cache/mod.rs
vendored
2
src/cache/mod.rs
vendored
@ -1 +1 @@
|
||||
pub mod cacher;
|
||||
pub mod cacher;
|
||||
|
@ -24,6 +24,14 @@ pub struct Config {
|
||||
pub binding_ip_addr: String,
|
||||
pub style: Style,
|
||||
pub redis_connection_url: String,
|
||||
pub aggregator: AggreatorConfig,
|
||||
}
|
||||
|
||||
/// Configuration options for the aggregator.
|
||||
#[derive(Clone)]
|
||||
pub struct AggreatorConfig {
|
||||
/// Whether to introduce a random delay before sending the request to the search engine.
|
||||
pub random_delay: bool,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
@ -45,6 +53,15 @@ impl Config {
|
||||
)?)
|
||||
.exec()?;
|
||||
|
||||
let production_use = globals.get::<_, bool>("production_use")?;
|
||||
let aggregator_config = if production_use {
|
||||
AggreatorConfig { random_delay: true }
|
||||
} else {
|
||||
AggreatorConfig {
|
||||
random_delay: false,
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Config {
|
||||
port: globals.get::<_, u16>("port")?,
|
||||
binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?,
|
||||
@ -53,6 +70,7 @@ impl Config {
|
||||
globals.get::<_, String>("colorscheme")?,
|
||||
),
|
||||
redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
|
||||
aggregator: aggregator_config,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
@ -116,7 +116,7 @@ impl RawSearchResult {
|
||||
}
|
||||
}
|
||||
|
||||
/// A named struct to store, serialize, deserialize the all the search results scraped and
|
||||
/// A named struct to store, serialize, deserialize the all the search results scraped and
|
||||
/// aggregated from the upstream search engines.
|
||||
///
|
||||
/// # Fields
|
||||
|
@ -29,6 +29,7 @@ use crate::engines::{duckduckgo, searx};
|
||||
///
|
||||
/// * `query` - Accepts a string to query with the above upstream search engines.
|
||||
/// * `page` - Accepts an u32 page number.
|
||||
/// * `random_delay` - Accepts a boolean value to add a random delay before making the request.
|
||||
///
|
||||
/// # Error
|
||||
///
|
||||
@ -38,14 +39,17 @@ use crate::engines::{duckduckgo, searx};
|
||||
pub async fn aggregate(
|
||||
query: &str,
|
||||
page: u32,
|
||||
random_delay: bool,
|
||||
) -> Result<SearchResults, Box<dyn std::error::Error>> {
|
||||
let user_agent: String = random_user_agent();
|
||||
let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
|
||||
|
||||
// Add a random delay before making the request.
|
||||
let mut rng = rand::thread_rng();
|
||||
let delay_secs = rng.gen_range(1..10);
|
||||
std::thread::sleep(Duration::from_secs(delay_secs));
|
||||
if random_delay {
|
||||
let mut rng = rand::thread_rng();
|
||||
let delay_secs = rng.gen_range(1..10);
|
||||
std::thread::sleep(Duration::from_secs(delay_secs));
|
||||
}
|
||||
|
||||
// fetch results from upstream search engines simultaneously/concurrently.
|
||||
let (ddg_map_results, searx_map_results) = join!(
|
||||
|
@ -81,10 +81,10 @@ pub async fn search(
|
||||
.insert_header(("location", "/"))
|
||||
.finish())
|
||||
} else {
|
||||
let page_url: String; // Declare the page_url variable without initializing it
|
||||
let page_url: String; // Declare the page_url variable without initializing it
|
||||
|
||||
// ...
|
||||
|
||||
|
||||
let page = match params.page {
|
||||
Some(page_number) => {
|
||||
if page_number <= 1 {
|
||||
@ -98,7 +98,7 @@ pub async fn search(
|
||||
"http://{}:{}/search?q={}&page={}",
|
||||
config.binding_ip_addr, config.port, query, page_number
|
||||
);
|
||||
|
||||
|
||||
page_number
|
||||
}
|
||||
}
|
||||
@ -110,11 +110,11 @@ pub async fn search(
|
||||
req.uri(),
|
||||
1
|
||||
);
|
||||
|
||||
|
||||
1
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// fetch the cached results json.
|
||||
let cached_results_json = redis_cache.cached_results_json(&page_url);
|
||||
// check if fetched results was indeed fetched or it was an error and if so
|
||||
@ -127,7 +127,7 @@ pub async fn search(
|
||||
}
|
||||
Err(_) => {
|
||||
let mut results_json: crate::search_results_handler::aggregation_models::SearchResults =
|
||||
aggregate(query, page).await?;
|
||||
aggregate(query, page, config.aggregator.random_delay).await?;
|
||||
results_json.add_style(config.style.clone());
|
||||
redis_cache
|
||||
.cache_results(serde_json::to_string(&results_json)?, &page_url)?;
|
||||
|
@ -19,3 +19,7 @@ theme = "simple" -- the theme name which should be used for the website
|
||||
|
||||
-- Caching
|
||||
redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.
|
||||
|
||||
production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users)
|
||||
-- if production_use is set to true
|
||||
-- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.
|
Loading…
Reference in New Issue
Block a user