From a912ac0724facb7ec871a06491d1a770b660b3c0 Mon Sep 17 00:00:00 2001 From: Ajeet Singh <34164838+Ajeets6@users.noreply.github.com> Date: Fri, 13 Oct 2023 21:00:38 +0530 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=93=9D=20Remove=20the=20two=20unneces?= =?UTF-8?q?sary=20lines=20from=20the=20code=20snippet=20(#312)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Removed unnecessary lines(issue:310) - # ports: - # - 6379:6379 * Update docs/installation.md Co-authored-by: neon_arch --------- Co-authored-by: neon_arch Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- docs/installation.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 1c4b112..8544dab 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -217,8 +217,6 @@ services: # Uncomment the following lines if you are using the `hybrid/latest` or `redis` image. # redis: # image: redis:latest - # ports: - # - 6379:6379 ``` Then make sure to edit the `docker-compose.yml` file as required. After that create a directory `websurfx` in the directory you have placed the `docker-compose.yml` file, and then in the new directory create two new empty files named `allowlist.txt` and `blocklist.txt`. Finally, create a new config file `config.lua` with the default configuration, which looks something like this: From ad297a0a96bd8a904d3606ec29fed49c517d0d27 Mon Sep 17 00:00:00 2001 From: leone Date: Sat, 14 Oct 2023 16:18:22 +0200 Subject: [PATCH 2/5] =?UTF-8?q?=E2=9C=A8=20Build=20arguments=20to=20make?= =?UTF-8?q?=20writing=20GitHub=20action=20for=20auto=20releasing=20new=20d?= =?UTF-8?q?ocker=20images=20easier=20=20(#324)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ feat: Dockerfile with arch and feature build arguments * 🛠️ fix: missing issuer certificate error * ⚙️ refactor: replace `--update` with new `--no-cache` option --- Cargo.toml | 8 +++++++- Dockerfile | 53 ++++++++++++++++++++++++++++------------------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2b12cac..b450db8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,8 +6,14 @@ description = "An open-source alternative to Searx that provides clean, ad-free, repository = "https://github.com/neon-mmd/websurfx" license = "AGPL-3.0" +[[bin]] +name = "websurfx" +test = true +bench = false +path = "src/bin/websurfx.rs" + [dependencies] -reqwest = {version="0.11.21",features=["json"]} +reqwest = {version="0.11.21",default-features = false,features = ["json", "rustls-tls"]} tokio = {version="1.32.0",features=["rt-multi-thread","macros"]} serde = {version="1.0.188",features=["derive"]} handlebars = { version = "4.4.0", features = ["dir_source"] } diff --git a/Dockerfile b/Dockerfile index 74c7adb..bc52ef7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,40 +1,43 @@ -FROM rust:latest AS chef +FROM --platform=$BUILDPLATFORM rust:1.73.0-alpine3.18 AS chef # We only pay the installation cost once, # it will be cached from the second build onwards +RUN apk add --no-cache alpine-sdk musl-dev g++ make libcrypto3 libressl-dev upx perl build-base RUN cargo install cargo-chef --locked WORKDIR /app FROM chef AS planner -COPY . . +COPY ./Cargo.toml ./Cargo.lock ./ RUN cargo chef prepare --recipe-path recipe.json -FROM chef AS builder +FROM --platform=$BUILDPLATFORM chef AS builder COPY --from=planner /app/recipe.json recipe.json -# Build dependencies - this is the caching Docker layer! -# Uncomment the line below if you want to use the `hybrid` caching feature. -# RUN cargo chef cook --release --features redis-cache --recipe-path recipe.json -# Comment the line below if you don't want to use the `In-Memory` caching feature. -RUN cargo chef cook --release --recipe-path recipe.json -# Uncomment the line below if you want to use the `no cache` feature. -# RUN cargo chef cook --release --no-default-features --recipe-path recipe.json -# Uncomment the line below if you want to use the `redis` caching feature. -# RUN cargo chef cook --release --no-default-features --features redis-cache --recipe-path recipe.json +# Specify the cache type to use (memory, redis, hybrid, no-cache) +ARG CACHE=memory +ENV CACHE=${CACHE} +# Cook the dependencies +RUN export ARCH=$(uname -m) && \ + if [ "$CACHE" = "memory" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --recipe-path recipe.json ; \ + else if [ "$CACHE" = "redis" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --no-default-features --features redis-cache --recipe-path recipe.json ; \ + else if [ "$CACHE" = "hybrid" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --features redis-cache --recipe-path recipe.json ; \ + else if [ "$CACHE" = "no-cache" ] ; then cargo chef cook --release --target=$ARCH-unknown-linux-musl --no-default-features --recipe-path recipe.json ; fi ; fi ; fi ; fi +# Copy the source code and public folder +COPY ./src ./src +COPY ./public ./public +# Build the application +RUN export ARCH=$(uname -m) && \ + if [ "$CACHE" = "memory" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl ; \ + else if [ "$CACHE" = "redis" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl --no-default-features --features redis-cache ; \ + else if [ "$CACHE" = "hybrid" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl --features redis-cache ; \ + else if [ "$CACHE" = "no-cache" ] ; then cargo build --release --target=$ARCH-unknown-linux-musl --no-default-features ; fi ; fi ; fi ; fi +# Optimise binary size with UPX +RUN export ARCH=$(uname -m) \ + && upx --lzma --best /app/target/$ARCH-unknown-linux-musl/release/websurfx \ + && cp /app/target/$ARCH-unknown-linux-musl/release/websurfx /usr/local/bin/websurfx -# Build application -COPY . . -# Uncomment the line below if you want to use the `hybrid` caching feature. -# RUN cargo install --path . --features redis-cache -# Comment the line below if you don't want to use the `In-Memory` caching feature. -RUN cargo install --path . -# Uncomment the line below if you want to use the `no cache` feature. -# RUN cargo install --path . --no-default-features -# Uncomment the line below if you want to use the `redis` caching feature. -# RUN cargo install --path . --no-default-features --features redis-cache -# We do not need the Rust toolchain to run the binary! -FROM gcr.io/distroless/cc-debian12 +FROM --platform=$BUILDPLATFORM scratch COPY --from=builder /app/public/ /opt/websurfx/public/ VOLUME ["/etc/xdg/websurfx/"] -COPY --from=builder /usr/local/cargo/bin/* /usr/local/bin/ +COPY --from=builder /usr/local/bin/websurfx /usr/local/bin/websurfx CMD ["websurfx"] From 092a38fc40e6e47f4b727c6e433103a31a9ded16 Mon Sep 17 00:00:00 2001 From: Ron Green <11993626+georgettica@users.noreply.github.com> Date: Sat, 14 Oct 2023 19:13:32 +0300 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=94=A7=20Remove=20the=20unwanted=20`o?= =?UTF-8?q?nce=5Fcell`=20crate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 132 +++++++++++++++++++++++++++++++++++++++++------------ Cargo.toml | 1 - 2 files changed, 104 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2074fd4..11b8343 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1563,6 +1563,20 @@ dependencies = [ "want 0.3.1", ] +[[package]] +name = "hyper-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +dependencies = [ + "futures-util", + "http 0.2.9", + "hyper 0.14.27", + "rustls", + "tokio 1.32.0", + "tokio-rustls", +] + [[package]] name = "hyper-tls" version = "0.3.2" @@ -1576,19 +1590,6 @@ dependencies = [ "tokio-io", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes 1.5.0", - "hyper 0.14.27", - "native-tls", - "tokio 1.32.0", - "tokio-native-tls", -] - [[package]] name = "idna" version = "0.1.5" @@ -2792,7 +2793,7 @@ dependencies = [ "futures 0.1.31", "http 0.1.21", "hyper 0.12.36", - "hyper-tls 0.3.2", + "hyper-tls", "log", "mime", "mime_guess", @@ -2826,29 +2827,46 @@ dependencies = [ "http 0.2.9", "http-body 0.4.5", "hyper 0.14.27", - "hyper-tls 0.5.0", + "hyper-rustls", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding 2.3.0", "pin-project-lite", + "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded 0.7.1", "system-configuration", "tokio 1.32.0", - "tokio-native-tls", + "tokio-rustls", "tower-service", "url 2.4.1", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg 0.50.0", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -2892,6 +2910,37 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustls" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +dependencies = [ + "base64 0.21.4", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rusty-hook" version = "0.11.2" @@ -2951,6 +3000,16 @@ dependencies = [ "tendril", ] +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.9.2" @@ -3202,6 +3261,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -3572,16 +3637,6 @@ dependencies = [ "syn 2.0.37", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio 1.32.0", -] - [[package]] name = "tokio-reactor" version = "0.1.12" @@ -3612,6 +3667,16 @@ dependencies = [ "tokio 1.32.0", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio 1.32.0", +] + [[package]] name = "tokio-sync" version = "0.1.8" @@ -3796,6 +3861,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "1.7.2" @@ -3963,6 +4034,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" + [[package]] name = "websurfx" version = "1.0.11" @@ -3985,7 +4062,6 @@ dependencies = [ "mimalloc", "mini-moka", "mlua", - "once_cell", "rand 0.8.5", "redis", "regex", diff --git a/Cargo.toml b/Cargo.toml index b450db8..6c4b287 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,6 @@ mlua = {version="0.9.1", features=["luajit", "vendored"]} redis = {version="0.23.3", features=["tokio-comp","connection-manager"], optional = true} md5 = {version="0.7.0"} rand={version="0.8.5"} -once_cell = {version="1.18.0"} error-stack = {version="0.4.0"} async-trait = {version="0.1.73"} regex = {version="1.9.4", features=["perf"]} From 405d00612f9c4baad128771b2dcfd0daf8a9a8f0 Mon Sep 17 00:00:00 2001 From: Uday Sagar <111575806+UdaySagar-Git@users.noreply.github.com> Date: Sun, 15 Oct 2023 00:44:28 +0530 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=94=A7=20Restricts=20the=20visibility?= =?UTF-8?q?=20of=20the=20`user-agent`=20helper=20module=20(#331)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/results/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/results/mod.rs b/src/results/mod.rs index 9ec3229..8c46af6 100644 --- a/src/results/mod.rs +++ b/src/results/mod.rs @@ -3,4 +3,4 @@ //! provides various models to aggregate search results into a standardized form. pub mod aggregator; -pub mod user_agent; +mod user_agent; From fc3b416970d14a5257a6a846a93abd54fbb503bd Mon Sep 17 00:00:00 2001 From: Aditya Phasu Date: Sun, 15 Oct 2023 03:37:59 +0530 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=94=A7=20remove=20unnecessary=20clone?= =?UTF-8?q?=20call=20(#333)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/results/aggregator.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index 8c9be2c..18fdb92 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -88,13 +88,7 @@ pub async fn aggregate( let query: String = query.to_owned(); tasks.push(tokio::spawn(async move { search_engine - .results( - &query, - page, - user_agent.clone(), - request_timeout, - safe_search, - ) + .results(&query, page, user_agent, request_timeout, safe_search) .await })); }