0
0
mirror of https://github.com/neon-mmd/websurfx.git synced 2024-11-21 21:48:21 -05:00

Fix git rebase problems, and the failing doctest

This commit is contained in:
Zsombor Gegesy 2023-09-14 20:26:08 +02:00
parent 1e7805cf42
commit e69126c5ea
6 changed files with 121 additions and 15 deletions

109
Cargo.lock generated
View File

@ -446,6 +446,12 @@ version = "3.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
[[package]]
name = "bytecount"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c"
[[package]] [[package]]
name = "byteorder" name = "byteorder"
version = "1.4.3" version = "1.4.3"
@ -478,6 +484,37 @@ dependencies = [
"bytes 1.5.0", "bytes 1.5.0",
] ]
[[package]]
name = "camino"
version = "1.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"
dependencies = [
"serde",
]
[[package]]
name = "cargo-platform"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cfa25e60aea747ec7e1124f238816749faa93759c6ff5b31f1ccdda137f4479"
dependencies = [
"serde",
]
[[package]]
name = "cargo_metadata"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
dependencies = [
"camino",
"cargo-platform",
"semver 1.0.18",
"serde",
"serde_json",
]
[[package]] [[package]]
name = "cast" name = "cast"
version = "0.3.0" version = "0.3.0"
@ -973,6 +1010,15 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "error-chain"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc"
dependencies = [
"version_check",
]
[[package]] [[package]]
name = "error-stack" name = "error-stack"
version = "0.4.1" version = "0.4.1"
@ -1256,6 +1302,12 @@ version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
[[package]]
name = "glob"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]] [[package]]
name = "governor" name = "governor"
version = "0.5.1" version = "0.5.1"
@ -1834,6 +1886,21 @@ dependencies = [
"unicase", "unicase",
] ]
[[package]]
name = "mini-moka"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23e0b72e7c9042467008b10279fc732326bd605459ae03bda88825909dd19b56"
dependencies = [
"crossbeam-channel",
"crossbeam-utils 0.8.16",
"dashmap",
"skeptic",
"smallvec 1.11.0",
"tagptr",
"triomphe",
]
[[package]] [[package]]
name = "miniz_oxide" name = "miniz_oxide"
version = "0.7.1" version = "0.7.1"
@ -2383,6 +2450,17 @@ dependencies = [
"url 2.4.1", "url 2.4.1",
] ]
[[package]]
name = "pulldown-cmark"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998"
dependencies = [
"bitflags 1.3.2",
"memchr",
"unicase",
]
[[package]] [[package]]
name = "quanta" name = "quanta"
version = "0.9.3" version = "0.9.3"
@ -2900,6 +2978,9 @@ name = "semver"
version = "1.0.18" version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "semver-parser" name = "semver-parser"
@ -3020,6 +3101,21 @@ version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
[[package]]
name = "skeptic"
version = "0.13.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8"
dependencies = [
"bytecount",
"cargo_metadata",
"error-chain",
"glob",
"pulldown-cmark",
"tempfile",
"walkdir",
]
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.9" version = "0.4.9"
@ -3197,6 +3293,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "tagptr"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]] [[package]]
name = "tempfile" name = "tempfile"
version = "3.8.0" version = "3.8.0"
@ -3553,6 +3655,12 @@ dependencies = [
"once_cell", "once_cell",
] ]
[[package]]
name = "triomphe"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee8098afad3fb0c54a9007aab6804558410503ad676d4633f9c2559a00ac0f"
[[package]] [[package]]
name = "try-lock" name = "try-lock"
version = "0.2.4" version = "0.2.4"
@ -3815,6 +3923,7 @@ dependencies = [
"log", "log",
"md5", "md5",
"mimalloc", "mimalloc",
"mini-moka",
"mlua", "mlua",
"once_cell", "once_cell",
"rand 0.8.5", "rand 0.8.5",

16
src/cache/cacher.rs vendored
View File

@ -8,7 +8,7 @@ use mini_moka::sync::Cache as MokaCache;
use std::time::Duration; use std::time::Duration;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use crate::{config::parser::Config, results::aggregation_models::SearchResults}; use crate::{config::parser::Config, models::aggregation_models::SearchResults};
use super::error::PoolError; use super::error::PoolError;
#[cfg(feature = "redis-cache")] #[cfg(feature = "redis-cache")]
@ -29,9 +29,9 @@ pub enum Cache {
impl Cache { impl Cache {
/// Builds the cache from the given configuration. /// Builds the cache from the given configuration.
pub async fn build(config: &Config) -> Self { pub async fn build(_config: &Config) -> Self {
#[cfg(feature = "redis-cache")] #[cfg(feature = "redis-cache")]
if let Some(url) = &config.redis_url { if let Some(url) = &_config.redis_url {
log::info!("Using Redis running at {} for caching", &url); log::info!("Using Redis running at {} for caching", &url);
return Cache::new( return Cache::new(
RedisCache::new(url, 5) RedisCache::new(url, 5)
@ -40,12 +40,15 @@ impl Cache {
); );
} }
#[cfg(feature = "memory-cache")] #[cfg(feature = "memory-cache")]
if config.in_memory_cache { {
log::info!("Using an in-memory cache"); log::info!("Using an in-memory cache");
return Cache::new_in_memory(); return Cache::new_in_memory();
} }
log::info!("Caching is disabled"); #[cfg(not(feature = "memory-cache"))]
Cache::Disabled {
log::info!("Caching is disabled");
Cache::Disabled
}
} }
/// Creates a new cache, which wraps the given RedisCache. /// Creates a new cache, which wraps the given RedisCache.
@ -117,6 +120,7 @@ impl Cache {
/// A structure to efficiently share the cache between threads - as it is protected by a Mutex. /// A structure to efficiently share the cache between threads - as it is protected by a Mutex.
pub struct SharedCache { pub struct SharedCache {
/// The internal cache protected from concurrent access by a mutex
cache: Mutex<Cache>, cache: Mutex<Cache>,
} }

View File

@ -20,8 +20,6 @@ pub struct Config {
/// It stores the redis connection url address on which the redis /// It stores the redis connection url address on which the redis
/// client should connect. /// client should connect.
pub redis_url: Option<String>, pub redis_url: Option<String>,
/// enable/disable the in-memory cache. Only checked, when no redis_url is provided.
pub in_memory_cache: bool,
/// It stores the option to whether enable or disable production use. /// It stores the option to whether enable or disable production use.
pub aggregator: AggregatorConfig, pub aggregator: AggregatorConfig,
/// It stores the option to whether enable or disable logs. /// It stores the option to whether enable or disable logs.
@ -102,10 +100,6 @@ impl Config {
globals.get::<_, String>("colorscheme")?, globals.get::<_, String>("colorscheme")?,
), ),
redis_url: globals.get::<_, String>("redis_url").ok(), redis_url: globals.get::<_, String>("redis_url").ok(),
in_memory_cache: globals
.get::<_, bool>("in_memory_cache")
.ok()
.unwrap_or(false),
aggregator: AggregatorConfig { aggregator: AggregatorConfig {
random_delay: globals.get::<_, bool>("production_use")?, random_delay: globals.get::<_, bool>("production_use")?,
}, },

View File

@ -40,7 +40,7 @@ use handler::paths::{file_path, FileType};
/// ///
/// ```rust /// ```rust
/// use std::net::TcpListener; /// use std::net::TcpListener;
/// use websurfx::{config::parser::Config, run}; /// use websurfx::{config::parser::Config, run, cache::cacher::Cache};
/// ///
/// let config = Config::parse(true).unwrap(); /// let config = Config::parse(true).unwrap();
/// let listener = TcpListener::bind("127.0.0.1:8080").expect("Failed to bind address"); /// let listener = TcpListener::bind("127.0.0.1:8080").expect("Failed to bind address");

View File

@ -47,7 +47,6 @@ theme = "simple" -- the theme name which should be used for the website
-- ### Caching ### -- ### Caching ###
redis_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on. redis_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.
in_memory_cache = true
-- ### Search Engines ### -- ### Search Engines ###
upstream_search_engines = { upstream_search_engines = {