mirror of
https://github.com/neon-mmd/websurfx.git
synced 2024-11-21 21:48:21 -05:00
Merge branch 'neon-mmd:rolling' into rolling
This commit is contained in:
Abhinav Pandey
GitHub
commit
25730ba5fa
132
Cargo.lock
generated
132
Cargo.lock
generated
@ -1563,6 +1563,20 @@ dependencies = [
|
||||
"want 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-rustls"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"http 0.2.9",
|
||||
"hyper 0.14.27",
|
||||
"rustls",
|
||||
"tokio 1.32.0",
|
||||
"tokio-rustls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-tls"
|
||||
version = "0.3.2"
|
||||
@ -1576,19 +1590,6 @@ dependencies = [
|
||||
"tokio-io",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-tls"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
|
||||
dependencies = [
|
||||
"bytes 1.5.0",
|
||||
"hyper 0.14.27",
|
||||
"native-tls",
|
||||
"tokio 1.32.0",
|
||||
"tokio-native-tls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.1.5"
|
||||
@ -2792,7 +2793,7 @@ dependencies = [
|
||||
"futures 0.1.31",
|
||||
"http 0.1.21",
|
||||
"hyper 0.12.36",
|
||||
"hyper-tls 0.3.2",
|
||||
"hyper-tls",
|
||||
"log",
|
||||
"mime",
|
||||
"mime_guess",
|
||||
@ -2826,29 +2827,46 @@ dependencies = [
|
||||
"http 0.2.9",
|
||||
"http-body 0.4.5",
|
||||
"hyper 0.14.27",
|
||||
"hyper-tls 0.5.0",
|
||||
"hyper-rustls",
|
||||
"ipnet",
|
||||
"js-sys",
|
||||
"log",
|
||||
"mime",
|
||||
"native-tls",
|
||||
"once_cell",
|
||||
"percent-encoding 2.3.0",
|
||||
"pin-project-lite",
|
||||
"rustls",
|
||||
"rustls-pemfile",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded 0.7.1",
|
||||
"system-configuration",
|
||||
"tokio 1.32.0",
|
||||
"tokio-native-tls",
|
||||
"tokio-rustls",
|
||||
"tower-service",
|
||||
"url 2.4.1",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"webpki-roots",
|
||||
"winreg 0.50.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.16.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"spin",
|
||||
"untrusted",
|
||||
"web-sys",
|
||||
"winapi 0.3.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.23"
|
||||
@ -2892,6 +2910,37 @@ dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.21.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8"
|
||||
dependencies = [
|
||||
"log",
|
||||
"ring",
|
||||
"rustls-webpki",
|
||||
"sct",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pemfile"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
|
||||
dependencies = [
|
||||
"base64 0.21.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.101.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusty-hook"
|
||||
version = "0.11.2"
|
||||
@ -2951,6 +3000,16 @@ dependencies = [
|
||||
"tendril",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sct"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "2.9.2"
|
||||
@ -3202,6 +3261,12 @@ dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.0"
|
||||
@ -3572,16 +3637,6 @@ dependencies = [
|
||||
"syn 2.0.37",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-native-tls"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
|
||||
dependencies = [
|
||||
"native-tls",
|
||||
"tokio 1.32.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-reactor"
|
||||
version = "0.1.12"
|
||||
@ -3612,6 +3667,16 @@ dependencies = [
|
||||
"tokio 1.32.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
|
||||
dependencies = [
|
||||
"rustls",
|
||||
"tokio 1.32.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-sync"
|
||||
version = "0.1.8"
|
||||
@ -3796,6 +3861,12 @@ version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "1.7.2"
|
||||
@ -3963,6 +4034,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.25.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc"
|
||||
|
||||
[[package]]
|
||||
name = "websurfx"
|
||||
version = "1.0.11"
|
||||
@ -3985,7 +4062,6 @@ dependencies = [
|
||||
"mimalloc",
|
||||
"mini-moka",
|
||||
"mlua",
|
||||
"once_cell",
|
||||
"rand 0.8.5",
|
||||
"redis",
|
||||
"regex",
|
||||
|
||||
@ -6,8 +6,14 @@ description = "An open-source alternative to Searx that provides clean, ad-free,
|
||||
repository = "https://github.com/neon-mmd/websurfx"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
[[bin]]
|
||||
name = "websurfx"
|
||||
test = true
|
||||
bench = false
|
||||
path = "src/bin/websurfx.rs"
|
||||
|
||||
[dependencies]
|
||||
reqwest = {version="0.11.21",features=["json"]}
|
||||
reqwest = {version="0.11.21",default-features = false,features = ["json", "rustls-tls"]}
|
||||
tokio = {version="1.32.0",features=["rt-multi-thread","macros"]}
|
||||
serde = {version="1.0.188",features=["derive"]}
|
||||
handlebars = { version = "4.4.0", features = ["dir_source"] }
|
||||
@ -23,7 +29,6 @@ mlua = {version="0.9.1", features=["luajit", "vendored"]}
|
||||
redis = {version="0.23.3", features=["tokio-comp","connection-manager"], optional = true}
|
||||
md5 = {version="0.7.0"}
|
||||
rand={version="0.8.5"}
|
||||
once_cell = {version="1.18.0"}
|
||||
error-stack = {version="0.4.0"}
|
||||
async-trait = {version="0.1.73"}
|
||||
regex = {version="1.9.4", features=["perf"]}
|
||||
|
||||
53
Dockerfile
53
Dockerfile
@ -1,40 +1,43 @@
|
||||
FROM rust:latest AS chef
|
||||
FROM --platform=$BUILDPLATFORM rust:1.73.0-alpine3.18 AS chef
|
||||
# We only pay the installation cost once,
|
||||
# it will be cached from the second build onwards
|
||||
RUN apk add --no-cache alpine-sdk musl-dev g++ make libcrypto3 libressl-dev upx perl build-base
|
||||
RUN cargo install cargo-chef --locked
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
FROM chef AS planner
|
||||
COPY . .
|
||||
COPY ./Cargo.toml ./Cargo.lock ./
|
||||
RUN cargo chef prepare --recipe-path recipe.json
|
||||
|
||||
FROM chef AS builder
|
||||
FROM --platform=$BUILDPLATFORM chef AS builder
|
||||
COPY --from=planner /app/recipe.json recipe.json
|
||||
# Build dependencies - this is the caching Docker layer!
|
||||
# Uncomment the line below if you want to use the `hybrid` caching feature.
|
||||
# RUN cargo chef cook --release --features redis-cache --recipe-path recipe.json
|
||||
# Comment the line below if you don't want to use the `In-Memory` caching feature.
|
||||
RUN cargo chef cook --release --recipe-path recipe.json
|
||||
# Uncomment the line below if you want to use the `no cache` feature.
|
||||
# RUN cargo chef cook --release --no-default-features --recipe-path recipe.json
|
||||
# Uncomment the line below if you want to use the `redis` caching feature.
|
||||
# RUN cargo chef cook --release --no-default-features --features redis-cache --recipe-path recipe.json
|
||||
# Specify the cache type to use (memory, redis, hybrid, no-cache)
|
||||
ARG CACHE=memory
|
||||
ENV CACHE=${CACHE}
|
||||
# Cook the dependencies
|
||||
RUN export ARCH=$(uname -m) && \
|
||||
if [ "$CACHE" = "memory" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --recipe-path recipe.json ; \
|
||||
else if [ "$CACHE" = "redis" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --no-default-features --features redis-cache --recipe-path recipe.json ; \
|
||||
else if [ "$CACHE" = "hybrid" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --features redis-cache --recipe-path recipe.json ; \
|
||||
else if [ "$CACHE" = "no-cache" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --no-default-features --recipe-path recipe.json ; fi ; fi ; fi ; fi
|
||||
# Copy the source code and public folder
|
||||
COPY ./src ./src
|
||||
COPY ./public ./public
|
||||
# Build the application
|
||||
RUN export ARCH=$(uname -m) && \
|
||||
if [ "$CACHE" = "memory" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl ; \
|
||||
else if [ "$CACHE" = "redis" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl --no-default-features --features redis-cache ; \
|
||||
else if [ "$CACHE" = "hybrid" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl --features redis-cache ; \
|
||||
else if [ "$CACHE" = "no-cache" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl --no-default-features ; fi ; fi ; fi ; fi
|
||||
# Optimise binary size with UPX
|
||||
RUN export ARCH=$(uname -m) \
|
||||
&& upx --lzma --best /app/target/$ARCH-unknown-linux-musl/release/websurfx \
|
||||
&& cp /app/target/$ARCH-unknown-linux-musl/release/websurfx /usr/local/bin/websurfx
|
||||
|
||||
# Build application
|
||||
COPY . .
|
||||
# Uncomment the line below if you want to use the `hybrid` caching feature.
|
||||
# RUN cargo install --path . --features redis-cache
|
||||
# Comment the line below if you don't want to use the `In-Memory` caching feature.
|
||||
RUN cargo install --path .
|
||||
# Uncomment the line below if you want to use the `no cache` feature.
|
||||
# RUN cargo install --path . --no-default-features
|
||||
# Uncomment the line below if you want to use the `redis` caching feature.
|
||||
# RUN cargo install --path . --no-default-features --features redis-cache
|
||||
|
||||
# We do not need the Rust toolchain to run the binary!
|
||||
FROM gcr.io/distroless/cc-debian12
|
||||
FROM --platform=$BUILDPLATFORM scratch
|
||||
COPY --from=builder /app/public/ /opt/websurfx/public/
|
||||
VOLUME ["/etc/xdg/websurfx/"]
|
||||
COPY --from=builder /usr/local/cargo/bin/* /usr/local/bin/
|
||||
COPY --from=builder /usr/local/bin/websurfx /usr/local/bin/websurfx
|
||||
CMD ["websurfx"]
|
||||
|
||||
@ -217,8 +217,6 @@ services:
|
||||
# Uncomment the following lines if you are using the `hybrid/latest` or `redis` image.
|
||||
# redis:
|
||||
# image: redis:latest
|
||||
# ports:
|
||||
# - 6379:6379
|
||||
```
|
||||
|
||||
Then make sure to edit the `docker-compose.yml` file as required. After that create a directory `websurfx` in the directory you have placed the `docker-compose.yml` file, and then in the new directory create two new empty files named `allowlist.txt` and `blocklist.txt`. Finally, create a new config file `config.lua` with the default configuration, which looks something like this:
|
||||
|
||||
@ -88,13 +88,7 @@ pub async fn aggregate(
|
||||
let query: String = query.to_owned();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
search_engine
|
||||
.results(
|
||||
&query,
|
||||
page,
|
||||
user_agent.clone(),
|
||||
request_timeout,
|
||||
safe_search,
|
||||
)
|
||||
.results(&query, page, user_agent, request_timeout, safe_search)
|
||||
.await
|
||||
}));
|
||||
}
|
||||
|
||||
@ -3,4 +3,4 @@
|
||||
//! provides various models to aggregate search results into a standardized form.
|
||||
|
||||
pub mod aggregator;
|
||||
pub mod user_agent;
|
||||
mod user_agent;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user