Merge branch 'neon-mmd:rolling' into rolling

Cargo.lock

@@ -1563,6 +1563,20 @@ dependencies = [
  "want 0.3.1",
 ]
 
+[[package]]
+name = "hyper-rustls"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97"
+dependencies = [
+ "futures-util",
+ "http 0.2.9",
+ "hyper 0.14.27",
+ "rustls",
+ "tokio 1.32.0",
+ "tokio-rustls",
+]
+
 [[package]]
 name = "hyper-tls"
 version = "0.3.2"
@@ -1576,19 +1590,6 @@ dependencies = [
  "tokio-io",
 ]
 
-[[package]]
-name = "hyper-tls"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
-dependencies = [
- "bytes 1.5.0",
- "hyper 0.14.27",
- "native-tls",
- "tokio 1.32.0",
- "tokio-native-tls",
-]
-
 [[package]]
 name = "idna"
 version = "0.1.5"
@@ -2792,7 +2793,7 @@ dependencies = [
  "futures 0.1.31",
  "http 0.1.21",
  "hyper 0.12.36",
- "hyper-tls 0.3.2",
+ "hyper-tls",
  "log",
  "mime",
  "mime_guess",
@@ -2826,29 +2827,46 @@ dependencies = [
  "http 0.2.9",
  "http-body 0.4.5",
  "hyper 0.14.27",
- "hyper-tls 0.5.0",
+ "hyper-rustls",
  "ipnet",
  "js-sys",
  "log",
  "mime",
- "native-tls",
  "once_cell",
  "percent-encoding 2.3.0",
  "pin-project-lite",
+ "rustls",
+ "rustls-pemfile",
  "serde",
  "serde_json",
  "serde_urlencoded 0.7.1",
  "system-configuration",
  "tokio 1.32.0",
- "tokio-native-tls",
+ "tokio-rustls",
  "tower-service",
  "url 2.4.1",
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "web-sys",
+ "webpki-roots",
  "winreg 0.50.0",
 ]
 
+[[package]]
+name = "ring"
+version = "0.16.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
+dependencies = [
+ "cc",
+ "libc",
+ "once_cell",
+ "spin",
+ "untrusted",
+ "web-sys",
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.23"
@@ -2892,6 +2910,37 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.21.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8"
+dependencies = [
+ "log",
+ "ring",
+ "rustls-webpki",
+ "sct",
+]
+
+[[package]]
+name = "rustls-pemfile"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
+dependencies = [
+ "base64 0.21.4",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.101.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "rusty-hook"
 version = "0.11.2"
@@ -2951,6 +3000,16 @@ dependencies = [
  "tendril",
 ]
 
+[[package]]
+name = "sct"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "security-framework"
 version = "2.9.2"
@@ -3202,6 +3261,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
 [[package]]
 name = "stable_deref_trait"
 version = "1.2.0"
@@ -3572,16 +3637,6 @@ dependencies = [
  "syn 2.0.37",
 ]
 
-[[package]]
-name = "tokio-native-tls"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
-dependencies = [
- "native-tls",
- "tokio 1.32.0",
-]
-
 [[package]]
 name = "tokio-reactor"
 version = "0.1.12"
@@ -3612,6 +3667,16 @@ dependencies = [
  "tokio 1.32.0",
 ]
 
+[[package]]
+name = "tokio-rustls"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
+dependencies = [
+ "rustls",
+ "tokio 1.32.0",
+]
+
 [[package]]
 name = "tokio-sync"
 version = "0.1.8"
@@ -3796,6 +3861,12 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
 
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
 [[package]]
 name = "url"
 version = "1.7.2"
@@ -3963,6 +4034,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "webpki-roots"
+version = "0.25.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc"
+
 [[package]]
 name = "websurfx"
 version = "1.0.11"
@@ -3985,7 +4062,6 @@ dependencies = [
  "mimalloc",
  "mini-moka",
  "mlua",
- "once_cell",
  "rand 0.8.5",
  "redis",
  "regex",

Cargo.toml

@@ -6,8 +6,14 @@ description = "An open-source alternative to Searx that provides clean, ad-free,
 repository = "https://github.com/neon-mmd/websurfx"
 license = "AGPL-3.0"
 
+[[bin]]
+name = "websurfx"
+test = true
+bench = false
+path = "src/bin/websurfx.rs"
+
 [dependencies]
-reqwest = {version="0.11.21",features=["json"]}
+reqwest = {version="0.11.21",default-features = false,features = ["json", "rustls-tls"]}
 tokio = {version="1.32.0",features=["rt-multi-thread","macros"]}
 serde = {version="1.0.188",features=["derive"]}
 handlebars = { version = "4.4.0", features = ["dir_source"] }
@@ -23,7 +29,6 @@ mlua = {version="0.9.1", features=["luajit", "vendored"]}
 redis = {version="0.23.3", features=["tokio-comp","connection-manager"], optional = true}
 md5 = {version="0.7.0"}
 rand={version="0.8.5"}
-once_cell = {version="1.18.0"}
 error-stack = {version="0.4.0"}
 async-trait = {version="0.1.73"}
 regex = {version="1.9.4", features=["perf"]}

Dockerfile

@@ -1,40 +1,43 @@
-FROM rust:latest AS chef
+FROM --platform=$BUILDPLATFORM rust:1.73.0-alpine3.18 AS chef
 # We only pay the installation cost once,
 # it will be cached from the second build onwards
+RUN apk add --no-cache alpine-sdk musl-dev g++ make libcrypto3 libressl-dev upx perl build-base
 RUN cargo install cargo-chef --locked
 
 WORKDIR /app
 
 FROM chef AS planner
-COPY . . 
+COPY ./Cargo.toml ./Cargo.lock ./
 RUN cargo chef prepare --recipe-path recipe.json
 
-FROM chef AS builder
+FROM --platform=$BUILDPLATFORM chef AS builder
 COPY --from=planner /app/recipe.json recipe.json
-# Build dependencies - this is the caching Docker layer!
+# Specify the cache type to use (memory, redis, hybrid, no-cache)
-# Uncomment the line below if you want to use the `hybrid` caching feature.
+ARG CACHE=memory
-# RUN cargo chef cook --release --features redis-cache --recipe-path recipe.json
+ENV CACHE=${CACHE}
-# Comment the line below if you don't want to use the `In-Memory` caching feature.
+# Cook the dependencies
-RUN cargo chef cook --release --recipe-path recipe.json
+RUN export ARCH=$(uname -m) && \
-# Uncomment the line below if you want to use the `no cache` feature.
+  if [ "$CACHE" = "memory" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --recipe-path recipe.json ; \
-# RUN cargo chef cook --release --no-default-features --recipe-path recipe.json
+  else if [ "$CACHE" = "redis" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --no-default-features --features redis-cache --recipe-path recipe.json ; \
-# Uncomment the line below if you want to use the `redis` caching feature.
+  else if [ "$CACHE" = "hybrid" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --features redis-cache --recipe-path recipe.json ; \
-# RUN cargo chef cook --release --no-default-features --features redis-cache --recipe-path recipe.json
+  else if [ "$CACHE" = "no-cache" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --no-default-features --recipe-path recipe.json ; fi ; fi ; fi ; fi
+# Copy the source code and public folder
+COPY ./src ./src
+COPY ./public ./public
+# Build the application
+RUN export ARCH=$(uname -m) && \
+  if [ "$CACHE" = "memory" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl ; \
+  else if [ "$CACHE" = "redis" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl --no-default-features --features redis-cache ; \
+  else if [ "$CACHE" = "hybrid" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl --features redis-cache ; \
+  else if [ "$CACHE" = "no-cache" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl --no-default-features ; fi ; fi ; fi ; fi
+# Optimise binary size with UPX
+RUN export ARCH=$(uname -m) \
+  && upx --lzma --best /app/target/$ARCH-unknown-linux-musl/release/websurfx \
+  && cp /app/target/$ARCH-unknown-linux-musl/release/websurfx /usr/local/bin/websurfx
 
-# Build application
-COPY . .
-# Uncomment the line below if you want to use the `hybrid` caching feature.
-# RUN cargo install --path . --features redis-cache
-# Comment the line below if you don't want to use the `In-Memory` caching feature.
-RUN cargo install --path .
-# Uncomment the line below if you want to use the `no cache` feature.
-# RUN cargo install --path . --no-default-features 
-# Uncomment the line below if you want to use the `redis` caching feature.
-# RUN cargo install --path . --no-default-features --features redis-cache
 
-# We do not need the Rust toolchain to run the binary!
+FROM --platform=$BUILDPLATFORM scratch
-FROM gcr.io/distroless/cc-debian12
 COPY --from=builder /app/public/ /opt/websurfx/public/
 VOLUME ["/etc/xdg/websurfx/"]
-COPY --from=builder /usr/local/cargo/bin/* /usr/local/bin/
+COPY --from=builder /usr/local/bin/websurfx /usr/local/bin/websurfx
 CMD ["websurfx"]

docs/installation.md

@@ -217,8 +217,6 @@ services:
   # Uncomment the following lines if you are using the `hybrid/latest` or `redis` image.
   # redis:
   #   image: redis:latest
-  #   ports:
-  #     - 6379:6379
 ```
 
 Then make sure to edit the `docker-compose.yml` file as required. After that create a directory `websurfx` in the directory you have placed the `docker-compose.yml` file, and then in the new directory create two new empty files named `allowlist.txt` and `blocklist.txt`. Finally, create a new config file `config.lua` with the default configuration, which looks something like this:

src/results/aggregator.rs

@@ -88,13 +88,7 @@ pub async fn aggregate(
         let query: String = query.to_owned();
         tasks.push(tokio::spawn(async move {
             search_engine
-                .results(
+                .results(&query, page, user_agent, request_timeout, safe_search)
-                    &query,
-                    page,
-                    user_agent.clone(),
-                    request_timeout,
-                    safe_search,
-                )
                 .await
         }));
     }

src/results/mod.rs

@@ -3,4 +3,4 @@
 //! provides various models to aggregate search results into a standardized form.
 
 pub mod aggregator;
-pub mod user_agent;
+mod user_agent;