From c3a90031fd81f9cc620ee9aba9522dbbf63f774b Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 2 Oct 2023 18:03:46 +0300 Subject: [PATCH 01/25] =?UTF-8?q?=F0=9F=A4=96=20ci:=20add=20dependabot=20a?= =?UTF-8?q?uto=20version=20bumping=20action?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8bfbb7e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "monthly" From 8787e06d7d95f8df8723fdae617111ea3001b3e8 Mon Sep 17 00:00:00 2001 From: alamin655 Date: Mon, 2 Oct 2023 22:00:24 +0530 Subject: [PATCH 02/25] =?UTF-8?q?=F0=9F=9A=80=20chore:=20Bump=20the=20app?= =?UTF-8?q?=20version=20(#290)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2984ffe..a5a81d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "1.0.0" +version = "1.0.1" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" From 055de8bd3cb832d5995247c1785817fd85378d47 Mon Sep 17 00:00:00 2001 From: alamin655 Date: Mon, 2 Oct 2023 22:00:59 +0530 Subject: [PATCH 03/25] =?UTF-8?q?=F0=9F=9A=80=20chore:=20Bump=20the=20app?= =?UTF-8?q?=20version=20(#290)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index c4ed1e3..d070cf8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3932,7 +3932,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "1.0.0" +version = "1.0.1" dependencies = [ "actix-cors", "actix-files", From b577f6909dcd088765643f935867d144f477c6a1 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Mon, 2 Oct 2023 19:54:22 +0300 Subject: [PATCH 04/25] =?UTF-8?q?=F0=9F=A4=96=20ci:=20enable=20auto-update?= =?UTF-8?q?s=20for=20docker,=20cargo=20and=20github=20actions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/dependabot.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8bfbb7e..dc1715a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,11 +1,14 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - version: 2 updates: - - package-ecosystem: "" # See documentation for possible values - directory: "/" # Location of package manifests + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "monthly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + - package-ecosystem: "docker" + directory: "/" schedule: interval: "monthly" From 6ec79144a08abce89fb296591745a170ad6538ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 03:30:56 +0000 Subject: [PATCH 05/25] build(deps): bump actions/setup-node from 2 to 3 Bumps [actions/setup-node](https://github.com/actions/setup-node) from 2 to 3. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index ab9f2fe..e014b39 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 with: node-version: '14' - uses: EddieHubCommunity/gh-action-open-source-labels@main From 4155a23e66555e3aac78156776e7261792323e8a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 03:32:07 +0000 Subject: [PATCH 06/25] build(deps): bump reqwest from 0.11.20 to 0.11.21 Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.20 to 0.11.21. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.20...v0.11.21) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 28 +++++++++++++++++++++++++--- Cargo.toml | 2 +- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d070cf8..e46b916 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2802,9 +2802,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "78fdbab6a7e1d7b13cc8ff10197f47986b41c639300cc3c8158cac7847c9bbef" dependencies = [ "base64 0.21.4", "bytes 1.5.0", @@ -2827,6 +2827,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.1", + "system-configuration", "tokio 1.32.0", "tokio-native-tls", "tower-service", @@ -3320,6 +3321,27 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -3956,7 +3978,7 @@ dependencies = [ "rand 0.8.5", "redis", "regex", - "reqwest 0.11.20", + "reqwest 0.11.21", "rusty-hook", "scraper", "serde", diff --git a/Cargo.toml b/Cargo.toml index a5a81d2..c02b2f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ repository = "https://github.com/neon-mmd/websurfx" license = "AGPL-3.0" [dependencies] -reqwest = {version="0.11.20",features=["json"]} +reqwest = {version="0.11.21",features=["json"]} tokio = {version="1.32.0",features=["rt-multi-thread","macros"]} serde = {version="1.0.188",features=["derive"]} handlebars = { version = "4.4.0", features = ["dir_source"] } From 2803471821a323cce8a596c66563acafdb4e1d34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 03:32:57 +0000 Subject: [PATCH 07/25] build(deps): bump mlua from 0.8.10 to 0.9.1 Bumps [mlua](https://github.com/khvzak/mlua) from 0.8.10 to 0.9.1. - [Changelog](https://github.com/khvzak/mlua/blob/master/CHANGELOG.md) - [Commits](https://github.com/khvzak/mlua/compare/v0.8.10...v0.9.1) --- updated-dependencies: - dependency-name: mlua dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 27 +++++++++++++++++++-------- Cargo.toml | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d070cf8..21dce61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -433,11 +433,12 @@ dependencies = [ [[package]] name = "bstr" -version = "0.2.17" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +checksum = "4c2f7349907b712260e64b0afe2f84692af14a454be26187d9df565c7f69266a" dependencies = [ "memchr", + "serde", ] [[package]] @@ -1992,20 +1993,30 @@ dependencies = [ [[package]] name = "mlua" -version = "0.8.10" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e" +checksum = "6c3a7a7ff4481ec91b951a733390211a8ace1caba57266ccb5f4d4966704e560" dependencies = [ "bstr", - "cc", - "lua-src", - "luajit-src", + "mlua-sys", "num-traits", "once_cell", - "pkg-config", "rustc-hash", ] +[[package]] +name = "mlua-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ec8b54eddb76093069cce9eeffb4c7b3a1a0fe66962d7bd44c4867928149ca3" +dependencies = [ + "cc", + "cfg-if 1.0.0", + "lua-src", + "luajit-src", + "pkg-config", +] + [[package]] name = "native-tls" version = "0.2.11" diff --git a/Cargo.toml b/Cargo.toml index a5a81d2..cf9df90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ serde_json = {version="1.0.105"} fake-useragent = {version="0.1.3"} env_logger = {version="0.10.0"} log = {version="0.4.20"} -mlua = {version="0.8.10", features=["luajit", "vendored"]} +mlua = {version="0.9.1", features=["luajit", "vendored"]} redis = {version="0.23.3", features=["tokio-comp","connection-manager"], optional = true} md5 = {version="0.7.0"} rand={version="0.8.5"} From 6cf4a63db7e0fbf31b73dd0ed2985876a4363f3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 05:36:44 +0000 Subject: [PATCH 08/25] build(deps): bump actions/stale from 5 to 8 Bumps [actions/stale](https://github.com/actions/stale) from 5 to 8. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v5...v8) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 5bae815..c4d68e0 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -19,7 +19,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v5 + - uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'Stale issue message' From 89e5895519deb671ac28d3d8e8b09dc6f117bce2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 05:44:41 +0000 Subject: [PATCH 09/25] build(deps): bump actions/checkout from 2 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Commits](https://github.com/actions/checkout/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/contributors.yml | 2 +- .github/workflows/mega-linter.yml | 2 +- .github/workflows/rust.yml | 4 ++-- .github/workflows/rust_format.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index d57a508..0ac59da 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 with: fetch-depth: 0 ref: ${{ github.event.repository.default_branch }} diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index c8f6cec..02bdaac 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -32,7 +32,7 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5d538ea..d82bde3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -25,7 +25,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends liblua5.4-dev liblua5.3-dev liblua5.2-dev liblua5.1-0-dev libluajit-5.1-dev - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: rustup toolchain install stable --profile minimal - uses: Swatinem/rust-cache@v2 with: @@ -39,7 +39,7 @@ jobs: cache-on-failure: '' cache-all-crates: '' save-if: '' - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} - name: Build run: cargo build --verbose diff --git a/.github/workflows/rust_format.yml b/.github/workflows/rust_format.yml index 1c1e16e..1b7c9fa 100644 --- a/.github/workflows/rust_format.yml +++ b/.github/workflows/rust_format.yml @@ -17,7 +17,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends liblua5.4-dev liblua5.3-dev liblua5.2-dev liblua5.1-0-dev libluajit-5.1-dev - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install minimal stable with clippy and rustfmt uses: actions-rs/toolchain@v1 with: From c324c7441199f69cd58595411d4e35eabf503621 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 05:53:06 +0000 Subject: [PATCH 10/25] build(deps): bump wow-actions/contributors-list from 1.1.0 to 1.2.0 Bumps [wow-actions/contributors-list](https://github.com/wow-actions/contributors-list) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/wow-actions/contributors-list/releases) - [Changelog](https://github.com/wow-actions/contributors-list/blob/master/CHANGELOG.md) - [Commits](https://github.com/wow-actions/contributors-list/compare/b9e91f91a51a55460fdcae64daad0cb8122cdd53...242b53835016268d20e79eeff6f42193c02be8c8) --- updated-dependencies: - dependency-name: wow-actions/contributors-list dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/contributors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 0ac59da..75aecf3 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -23,7 +23,7 @@ jobs: ref: ${{ github.event.repository.default_branch }} - name: Update contributors list - uses: wow-actions/contributors-list@b9e91f91a51a55460fdcae64daad0cb8122cdd53 # v1.1.0 + uses: wow-actions/contributors-list@242b53835016268d20e79eeff6f42193c02be8c8 # v1.2.0 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} svgPath: images/contributors_list.svg From 0fbb52308231c6c3d40f9c7d7a97e438697783f7 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 3 Oct 2023 20:52:16 +0300 Subject: [PATCH 11/25] ci(Mergify): configuration update Signed-off-by: neon_arch --- .mergify.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .mergify.yml diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 0000000..8c2f886 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,18 @@ +pull_request_rules: + - name: Automatic merge on approval + conditions: + - "status-success=checks/approved" + - "#approved-reviews-by>=2" + actions: + queue: + method: squash + - name: automatic update of pull requests where more 5 commits behind + conditions: + - "#commits-behind>5" + actions: + update: + - name: delete head branch after merge + conditions: + - merged + actions: + delete_head_branch: {} \ No newline at end of file From a9f54b40275283fff88f4bf9ecc9dff51c64f5f8 Mon Sep 17 00:00:00 2001 From: neon_arch Date: Tue, 3 Oct 2023 21:11:29 +0300 Subject: [PATCH 12/25] =?UTF-8?q?=F0=9F=9A=80=20chore:=20bump=20the=20app?= =?UTF-8?q?=20version=20(#299)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e46b916..dcba881 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -448,9 +448,9 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytecount" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" +checksum = "ad152d03a2c813c80bb94fedbf3a3f02b28f793e39e7c214c8a0bcc196343de7" [[package]] name = "byteorder" @@ -991,9 +991,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" dependencies = [ "errno-dragonfly", "libc", @@ -1866,9 +1866,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -2802,9 +2802,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.21" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78fdbab6a7e1d7b13cc8ff10197f47986b41c639300cc3c8158cac7847c9bbef" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "base64 0.21.4", "bytes 1.5.0", @@ -3954,7 +3954,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "1.0.1" +version = "1.0.7" dependencies = [ "actix-cors", "actix-files", @@ -3978,7 +3978,7 @@ dependencies = [ "rand 0.8.5", "redis", "regex", - "reqwest 0.11.21", + "reqwest 0.11.22", "rusty-hook", "scraper", "serde", diff --git a/Cargo.toml b/Cargo.toml index c02b2f8..db0fab2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "1.0.1" +version = "1.0.7" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" From 6bc434cebd70863cc2d719bdfe1e2accb6cfce32 Mon Sep 17 00:00:00 2001 From: Niloth P <20315308+Niloth-p@users.noreply.github.com> Date: Wed, 4 Oct 2023 01:00:19 +0530 Subject: [PATCH 13/25] docs(readme): Add Codefactor badge --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 6bdfc02..7245849 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,12 @@ src="https://img.shields.io/badge/Gitpod-Ready--to--Code-blue?logo=gitpod" /> + CodeFactor +

From 5d26a2ec6d13fff0896df47780e9225e553dfb89 Mon Sep 17 00:00:00 2001 From: alamin655 Date: Wed, 4 Oct 2023 09:17:21 +0530 Subject: [PATCH 14/25] =?UTF-8?q?=F0=9F=93=9D=20Codefactor=20code=20covera?= =?UTF-8?q?ge=20shield=20badge=20(#301)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7245849..77789c4 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,11 @@ src="https://img.shields.io/badge/Gitpod-Ready--to--Code-blue?logo=gitpod" /> - CodeFactor - + />

From 57410378a4ab14cbc9f633b74bf7825d35a08462 Mon Sep 17 00:00:00 2001 From: alamin655 Date: Wed, 4 Oct 2023 09:28:05 +0530 Subject: [PATCH 15/25] =?UTF-8?q?=F0=9F=93=9D=20Codefactor=20code=20covera?= =?UTF-8?q?ge=20shield=20badge=20(#301)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 77789c4..6161143 100644 --- a/README.md +++ b/README.md @@ -35,17 +35,18 @@ src="https://img.shields.io/maintenance/yes/2023?style=flat-square" /> + + CodeFactor + Gitpod - CodeFactor

From 074975bc20e97257651feac2a8a43ed3e4a89edf Mon Sep 17 00:00:00 2001 From: alamin655 Date: Wed, 4 Oct 2023 09:29:18 +0530 Subject: [PATCH 16/25] =?UTF-8?q?=F0=9F=9A=80=20chore:=20bump=20the=20app?= =?UTF-8?q?=20version=20(#301)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index db0fab2..cdeffa5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "1.0.7" +version = "1.0.8" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" From 01cb86deddc12332cc8eb206e01253b9523d6245 Mon Sep 17 00:00:00 2001 From: alamin655 Date: Wed, 4 Oct 2023 09:29:47 +0530 Subject: [PATCH 17/25] =?UTF-8?q?=F0=9F=9A=80=20chore:=20bump=20the=20app?= =?UTF-8?q?=20version=20(#301)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index dcba881..6bcb939 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3954,7 +3954,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "1.0.7" +version = "1.0.8" dependencies = [ "actix-cors", "actix-files", From 5e63f37d706e9182996fc42a4748914d3ca50f55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Mota?= Date: Wed, 4 Oct 2023 19:17:42 +0100 Subject: [PATCH 18/25] =?UTF-8?q?=F0=9F=93=9D=20Three=20points=20listed=20?= =?UTF-8?q?under=20the=20`Features`=20section=20(#304)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add additional features to README * 🚀 chore: bump the app version (#304) * 🚀 chore: bump the app version (#304) --------- Co-authored-by: alamin655 --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6bcb939..1d45c38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3954,7 +3954,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "1.0.8" +version = "1.0.9" dependencies = [ "actix-cors", "actix-files", diff --git a/Cargo.toml b/Cargo.toml index cdeffa5..a291449 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "1.0.8" +version = "1.0.9" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" diff --git a/README.md b/README.md index 6161143..b5b7e3b 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,9 @@ # Features 🚀 - 🎨 Make Websurfx uniquely yours with twelve color schemes provided by default. It also supports creation of custom themes and color schemes in a quick and easy way, so unleash your creativity! +- 🚀 Easy to setup with docker or on bare metal with various installation/deployement options. +- ⛔ Search filtering to filter search results based on four different levels. +- 💾 Different caching levels focusing on reliability, speed and resiliancy. - 🔐 Fast, private, and secure - 🆓 100% free and open source - 💨 Ad-free and clean results From 75a77d25f06f2dbf53494655f76719568d7adc9f Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Sun, 24 Sep 2023 13:54:08 +0200 Subject: [PATCH 19/25] Create separate search_result_parser --- src/config/parser.rs | 2 +- src/engines/duckduckgo.rs | 49 ++++++++++++++------------- src/engines/mod.rs | 1 + src/engines/search_result_parser.rs | 38 +++++++++++++++++++++ src/engines/searx.rs | 52 ++++++++++++++--------------- src/models/aggregation_models.rs | 2 ++ src/models/engine_models.rs | 26 ++++++++++----- src/server/routes/search.rs | 2 +- 8 files changed, 110 insertions(+), 62 deletions(-) create mode 100644 src/engines/search_result_parser.rs diff --git a/src/config/parser.rs b/src/config/parser.rs index fb9f8b1..d7202a6 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -111,7 +111,7 @@ impl Config { .get::<_, HashMap>("upstream_search_engines")? .into_iter() .filter_map(|(key, value)| value.then_some(key)) - .filter_map(|engine| crate::models::engine_models::EngineHandler::new(&engine)) + .filter_map(|engine| crate::models::engine_models::EngineHandler::new(&engine).ok()) .collect(), request_timeout: globals.get::<_, u8>("request_timeout")?, threads, diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index 0f06ea4..318e764 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use reqwest::header::HeaderMap; -use scraper::{Html, Selector}; +use scraper::Html; use crate::models::aggregation_models::SearchResult; @@ -13,9 +13,27 @@ use crate::models::engine_models::{EngineError, SearchEngine}; use error_stack::{Report, Result, ResultExt}; +use super::search_result_parser::SearchResultParser; + /// A new DuckDuckGo engine type defined in-order to implement the `SearchEngine` trait which allows to /// reduce code duplication as well as allows to create vector of different search engines easily. -pub struct DuckDuckGo; +pub struct DuckDuckGo { + parser: SearchResultParser, +} + +impl DuckDuckGo { + pub fn new() -> Result { + Ok(Self { + parser: SearchResultParser::new( + ".no-results", + ".result", + ".result__a", + ".result__url", + ".result__snippet", + )?, + }) + } +} #[async_trait::async_trait] impl SearchEngine for DuckDuckGo { @@ -59,34 +77,17 @@ impl SearchEngine for DuckDuckGo { &DuckDuckGo::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?, ); - let no_result: Selector = Selector::parse(".no-results") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".no-results"))?; - - if document.select(&no_result).next().is_some() { + if document.select(&self.parser.no_result).next().is_some() { return Err(Report::new(EngineError::EmptyResultSet)); } - let results: Selector = Selector::parse(".result") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result"))?; - let result_title: Selector = Selector::parse(".result__a") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result__a"))?; - let result_url: Selector = Selector::parse(".result__url") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result__url"))?; - let result_desc: Selector = Selector::parse(".result__snippet") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result__snippet"))?; - // scrape all the results from the html Ok(document - .select(&results) + .select(&self.parser.results) .map(|result| { SearchResult::new( result - .select(&result_title) + .select(&self.parser.result_title) .next() .unwrap() .inner_html() @@ -94,7 +95,7 @@ impl SearchEngine for DuckDuckGo { format!( "https://{}", result - .select(&result_url) + .select(&self.parser.result_url) .next() .unwrap() .inner_html() @@ -102,7 +103,7 @@ impl SearchEngine for DuckDuckGo { ) .as_str(), result - .select(&result_desc) + .select(&self.parser.result_desc) .next() .unwrap() .inner_html() diff --git a/src/engines/mod.rs b/src/engines/mod.rs index 0016728..39b50c8 100644 --- a/src/engines/mod.rs +++ b/src/engines/mod.rs @@ -4,4 +4,5 @@ //! code. Moreover, it also provides a custom error for the upstream search engine handling code. pub mod duckduckgo; +pub mod search_result_parser; pub mod searx; diff --git a/src/engines/search_result_parser.rs b/src/engines/search_result_parser.rs new file mode 100644 index 0000000..6918917 --- /dev/null +++ b/src/engines/search_result_parser.rs @@ -0,0 +1,38 @@ +use crate::models::engine_models::EngineError; +use error_stack::{Report, Result, ResultExt}; +use scraper::{Html, Selector}; + +pub struct SearchResultParser { + pub no_result: Selector, + pub results: Selector, + pub result_title: Selector, + pub result_url: Selector, + pub result_desc: Selector, +} + +impl SearchResultParser { + pub fn new( + no_result_selector: &str, + results_selector: &str, + result_title_selector: &str, + result_url_selector: &str, + result_desc_selector: &str, + ) -> Result { + Ok(SearchResultParser { + no_result: new_selector(no_result_selector)?, + results: new_selector(results_selector)?, + result_title: new_selector(result_title_selector)?, + result_url: new_selector(result_url_selector)?, + result_desc: new_selector(result_desc_selector)?, + }) + } +} + +fn new_selector(selector: &str) -> Result { + Selector::parse(selector).map_err(|err| { + Report::new(EngineError::UnexpectedError).attach_printable(format!( + "invalid CSS selector: {}, err: {:?}", + selector, err + )) + }) +} diff --git a/src/engines/searx.rs b/src/engines/searx.rs index 6ab0469..32f286e 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -3,16 +3,34 @@ //! number if provided. use reqwest::header::HeaderMap; -use scraper::{Html, Selector}; +use scraper::Html; use std::collections::HashMap; +use super::search_result_parser::SearchResultParser; use crate::models::aggregation_models::SearchResult; use crate::models::engine_models::{EngineError, SearchEngine}; use error_stack::{Report, Result, ResultExt}; /// A new Searx engine type defined in-order to implement the `SearchEngine` trait which allows to /// reduce code duplication as well as allows to create vector of different search engines easily. -pub struct Searx; +pub struct Searx { + parser: SearchResultParser, +} + +impl Searx { + // new Searchx engine + pub fn new() -> Result { + Ok(Self { + parser: SearchResultParser::new( + "#urls>.dialog-error>p", + ".result", + "h3>a", + "h3>a", + ".content", + )?, + }) + } +} #[async_trait::async_trait] impl SearchEngine for Searx { @@ -52,13 +70,7 @@ impl SearchEngine for Searx { &Searx::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?, ); - let no_result: Selector = Selector::parse("#urls>.dialog-error>p") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| { - format!("invalid CSS selector: {}", "#urls>.dialog-error>p") - })?; - - if let Some(no_result_msg) = document.select(&no_result).nth(1) { + if let Some(no_result_msg) = document.select(&self.parser.no_result).nth(1) { if no_result_msg.inner_html() == "we didn't find any results. Please use another query or search in more categories" { @@ -66,40 +78,26 @@ impl SearchEngine for Searx { } } - let results: Selector = Selector::parse(".result") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".result"))?; - let result_title: Selector = Selector::parse("h3>a") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", "h3>a"))?; - let result_url: Selector = Selector::parse("h3>a") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", "h3>a"))?; - - let result_desc: Selector = Selector::parse(".content") - .map_err(|_| Report::new(EngineError::UnexpectedError)) - .attach_printable_lazy(|| format!("invalid CSS selector: {}", ".content"))?; - // scrape all the results from the html Ok(document - .select(&results) + .select(&self.parser.results) .map(|result| { SearchResult::new( result - .select(&result_title) + .select(&self.parser.result_title) .next() .unwrap() .inner_html() .trim(), result - .select(&result_url) + .select(&self.parser.result_url) .next() .unwrap() .value() .attr("href") .unwrap(), result - .select(&result_desc) + .select(&self.parser.result_desc) .next() .unwrap() .inner_html() diff --git a/src/models/aggregation_models.rs b/src/models/aggregation_models.rs index 72bbf08..6e4bddf 100644 --- a/src/models/aggregation_models.rs +++ b/src/models/aggregation_models.rs @@ -85,12 +85,14 @@ impl EngineErrorInfo { pub fn new(error: &EngineError, engine: &str) -> Self { Self { error: match error { + EngineError::EngineNotFound => "EngineNotFound".to_owned(), EngineError::RequestError => "RequestError".to_owned(), EngineError::EmptyResultSet => "EmptyResultSet".to_owned(), EngineError::UnexpectedError => "UnexpectedError".to_owned(), }, engine: engine.to_owned(), severity_color: match error { + EngineError::EngineNotFound => "red".to_owned(), EngineError::RequestError => "green".to_owned(), EngineError::EmptyResultSet => "blue".to_owned(), EngineError::UnexpectedError => "red".to_owned(), diff --git a/src/models/engine_models.rs b/src/models/engine_models.rs index d4a4e72..77ec4c4 100644 --- a/src/models/engine_models.rs +++ b/src/models/engine_models.rs @@ -2,12 +2,14 @@ //! the upstream search engines with the search query provided by the user. use super::aggregation_models::SearchResult; -use error_stack::{Result, ResultExt}; +use error_stack::{Report, Result, ResultExt}; use std::{collections::HashMap, fmt, time::Duration}; /// A custom error type used for handle engine associated errors. #[derive(Debug)] pub enum EngineError { + // No matching engine found + EngineNotFound, /// This variant handles all request related errors like forbidden, not found, /// etc. EmptyResultSet, @@ -24,6 +26,9 @@ pub enum EngineError { impl fmt::Display for EngineError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + EngineError::EngineNotFound => { + write!(f, "Search engine not found") + } EngineError::EmptyResultSet => { write!(f, "The upstream search engine returned an empty result set") } @@ -134,18 +139,21 @@ impl EngineHandler { /// # Returns /// /// It returns an option either containing the value or a none if the engine is unknown - pub fn new(engine_name: &str) -> Option { + pub fn new(engine_name: &str) -> Result { let engine: (&'static str, Box) = match engine_name.to_lowercase().as_str() { - "duckduckgo" => ( - "duckduckgo", - Box::new(crate::engines::duckduckgo::DuckDuckGo), - ), - "searx" => ("searx", Box::new(crate::engines::searx::Searx)), - _ => return None, + "duckduckgo" => { + let engine = crate::engines::duckduckgo::DuckDuckGo::new()?; + ("duckduckgo", Box::new(engine)) + } + "searx" => { + let engine = crate::engines::searx::Searx::new()?; + ("searx", Box::new(engine)) + } + _ => return Err(Report::from(EngineError::EngineNotFound)), }; - Some(Self { + Ok(Self { engine: engine.1, name: engine.0, }) diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index 80db98f..9dbd1e1 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -191,7 +191,7 @@ async fn results( let engines: Vec = cookie_value .engines .iter() - .filter_map(|name| EngineHandler::new(name)) + .filter_map(|name| EngineHandler::new(name).ok()) .collect(); safe_search_level = match config.safe_search { From 57c73d38c80024bcf9daf19ed9b7e2af6b1190fd Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Sun, 24 Sep 2023 15:09:03 +0200 Subject: [PATCH 20/25] Refactor the search result parsing --- src/engines/duckduckgo.rs | 40 ++++++----------------- src/engines/search_result_parser.rs | 49 ++++++++++++++++++++++++----- src/engines/searx.rs | 42 ++++++++----------------- src/models/engine_models.rs | 2 +- 4 files changed, 65 insertions(+), 68 deletions(-) diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index 318e764..8a28dda 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -18,10 +18,12 @@ use super::search_result_parser::SearchResultParser; /// A new DuckDuckGo engine type defined in-order to implement the `SearchEngine` trait which allows to /// reduce code duplication as well as allows to create vector of different search engines easily. pub struct DuckDuckGo { + // The parser, used to interpret the search result. parser: SearchResultParser, } impl DuckDuckGo { + /// Creates the DuckDuckGo parser. pub fn new() -> Result { Ok(Self { parser: SearchResultParser::new( @@ -77,41 +79,19 @@ impl SearchEngine for DuckDuckGo { &DuckDuckGo::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?, ); - if document.select(&self.parser.no_result).next().is_some() { + if self.parser.parse_for_no_results(&document).next().is_some() { return Err(Report::new(EngineError::EmptyResultSet)); } // scrape all the results from the html - Ok(document - .select(&self.parser.results) - .map(|result| { - SearchResult::new( - result - .select(&self.parser.result_title) - .next() - .unwrap() - .inner_html() - .trim(), - format!( - "https://{}", - result - .select(&self.parser.result_url) - .next() - .unwrap() - .inner_html() - .trim() - ) - .as_str(), - result - .select(&self.parser.result_desc) - .next() - .unwrap() - .inner_html() - .trim(), + self.parser + .parse_for_results(&document, |title, url, desc| { + Some(SearchResult::new( + title.inner_html().trim(), + &format!("https://{}", url.inner_html().trim()), + desc.inner_html().trim(), &["duckduckgo"], - ) + )) }) - .map(|search_result| (search_result.url.clone(), search_result)) - .collect()) } } diff --git a/src/engines/search_result_parser.rs b/src/engines/search_result_parser.rs index 6918917..94fe0e8 100644 --- a/src/engines/search_result_parser.rs +++ b/src/engines/search_result_parser.rs @@ -1,16 +1,21 @@ -use crate::models::engine_models::EngineError; -use error_stack::{Report, Result, ResultExt}; -use scraper::{Html, Selector}; +//! This modules provides helper functionalities for parsing a html document into internal SearchResult. +use std::collections::HashMap; +use crate::models::{aggregation_models::SearchResult, engine_models::EngineError}; +use error_stack::{Report, Result}; +use scraper::{html::Select, ElementRef, Html, Selector}; + +/// A html search result parser, based on a predefined CSS selectors. pub struct SearchResultParser { - pub no_result: Selector, - pub results: Selector, - pub result_title: Selector, - pub result_url: Selector, - pub result_desc: Selector, + no_result: Selector, + results: Selector, + result_title: Selector, + result_url: Selector, + result_desc: Selector, } impl SearchResultParser { + /// Creates a new parser, if all the selectors are valid, otherwise it returns an EngineError pub fn new( no_result_selector: &str, results_selector: &str, @@ -26,8 +31,36 @@ impl SearchResultParser { result_desc: new_selector(result_desc_selector)?, }) } + + /// Parse the html and returns element representing the 'no result found' response. + pub fn parse_for_no_results<'a>(&'a self, document: &'a Html) -> Select<'a, 'a> { + document.select(&self.no_result) + } + + /// Parse the html, and convert the results to SearchResult with the help of the builder function + pub fn parse_for_results( + &self, + document: &Html, + builder: impl Fn(&ElementRef<'_>, &ElementRef<'_>, &ElementRef<'_>) -> Option, + ) -> Result, EngineError> { + let res = document + .select(&self.results) + .filter_map(|result| { + let title = result.select(&self.result_title).next(); + let url = result.select(&self.result_url).next(); + let desc = result.select(&self.result_desc).next(); + match (title, url, desc) { + (Some(ref t), Some(ref u), Some(ref d)) => builder(t, u, d), + _ => None, + } + }) + .map(|search_result| (search_result.url.clone(), search_result)) + .collect(); + Ok(res) + } } +/// Create a Selector struct, if the given parameter is a valid css expression, otherwise convert it into an EngineError. fn new_selector(selector: &str) -> Result { Selector::parse(selector).map_err(|err| { Report::new(EngineError::UnexpectedError).attach_printable(format!( diff --git a/src/engines/searx.rs b/src/engines/searx.rs index 32f286e..ca08b98 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -14,11 +14,12 @@ use error_stack::{Report, Result, ResultExt}; /// A new Searx engine type defined in-order to implement the `SearchEngine` trait which allows to /// reduce code duplication as well as allows to create vector of different search engines easily. pub struct Searx { + // The parser, used to interpret the search result. parser: SearchResultParser, } impl Searx { - // new Searchx engine + /// creates a Searx parser pub fn new() -> Result { Ok(Self { parser: SearchResultParser::new( @@ -70,7 +71,7 @@ impl SearchEngine for Searx { &Searx::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?, ); - if let Some(no_result_msg) = document.select(&self.parser.no_result).nth(1) { + if let Some(no_result_msg) = self.parser.parse_for_no_results(&document).nth(1) { if no_result_msg.inner_html() == "we didn't find any results. Please use another query or search in more categories" { @@ -79,33 +80,16 @@ impl SearchEngine for Searx { } // scrape all the results from the html - Ok(document - .select(&self.parser.results) - .map(|result| { - SearchResult::new( - result - .select(&self.parser.result_title) - .next() - .unwrap() - .inner_html() - .trim(), - result - .select(&self.parser.result_url) - .next() - .unwrap() - .value() - .attr("href") - .unwrap(), - result - .select(&self.parser.result_desc) - .next() - .unwrap() - .inner_html() - .trim(), - &["searx"], - ) + self.parser + .parse_for_results(&document, |title, url, desc| { + url.value().attr("href").map(|url| { + SearchResult::new( + title.inner_html().trim(), + url, + desc.inner_html().trim(), + &["searx"], + ) + }) }) - .map(|search_result| (search_result.url.clone(), search_result)) - .collect()) } } diff --git a/src/models/engine_models.rs b/src/models/engine_models.rs index 77ec4c4..d53fc0c 100644 --- a/src/models/engine_models.rs +++ b/src/models/engine_models.rs @@ -8,7 +8,7 @@ use std::{collections::HashMap, fmt, time::Duration}; /// A custom error type used for handle engine associated errors. #[derive(Debug)] pub enum EngineError { - // No matching engine found + /// No matching engine found EngineNotFound, /// This variant handles all request related errors like forbidden, not found, /// etc. From 32abacb4c30d1c3756d2e5d5a15a7818445b1ddd Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Mon, 25 Sep 2023 00:00:06 +0200 Subject: [PATCH 21/25] Fix missing doc errors --- src/engines/duckduckgo.rs | 2 +- src/engines/search_result_parser.rs | 5 +++++ src/engines/searx.rs | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/engines/duckduckgo.rs b/src/engines/duckduckgo.rs index 8a28dda..352a33b 100644 --- a/src/engines/duckduckgo.rs +++ b/src/engines/duckduckgo.rs @@ -18,7 +18,7 @@ use super::search_result_parser::SearchResultParser; /// A new DuckDuckGo engine type defined in-order to implement the `SearchEngine` trait which allows to /// reduce code duplication as well as allows to create vector of different search engines easily. pub struct DuckDuckGo { - // The parser, used to interpret the search result. + /// The parser, used to interpret the search result. parser: SearchResultParser, } diff --git a/src/engines/search_result_parser.rs b/src/engines/search_result_parser.rs index 94fe0e8..0512bdd 100644 --- a/src/engines/search_result_parser.rs +++ b/src/engines/search_result_parser.rs @@ -7,10 +7,15 @@ use scraper::{html::Select, ElementRef, Html, Selector}; /// A html search result parser, based on a predefined CSS selectors. pub struct SearchResultParser { + /// selector to locate the element which is displayed, if there were nothing found. no_result: Selector, + /// selector to locate the element which contains one item from the search result. results: Selector, + /// selector to locate the title relative to the search result item. result_title: Selector, + /// selector to locate the url relative to the search result item. result_url: Selector, + /// selector to locate the description relative to the search result item. result_desc: Selector, } diff --git a/src/engines/searx.rs b/src/engines/searx.rs index ca08b98..79c1e95 100644 --- a/src/engines/searx.rs +++ b/src/engines/searx.rs @@ -14,7 +14,7 @@ use error_stack::{Report, Result, ResultExt}; /// A new Searx engine type defined in-order to implement the `SearchEngine` trait which allows to /// reduce code duplication as well as allows to create vector of different search engines easily. pub struct Searx { - // The parser, used to interpret the search result. + /// The parser, used to interpret the search result. parser: SearchResultParser, } From 8ed4c9e206b10bdc0d8ae59d160cadc24ada8d53 Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Sat, 7 Oct 2023 10:31:58 +0200 Subject: [PATCH 22/25] Propagate errors upward, if an engine can't be initialized --- src/config/parser.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/config/parser.rs b/src/config/parser.rs index d7202a6..0acdd25 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -3,6 +3,7 @@ use crate::handler::paths::{file_path, FileType}; +use crate::models::engine_models::{EngineError, EngineHandler}; use crate::models::parser_models::{AggregatorConfig, RateLimiter, Style}; use log::LevelFilter; use mlua::Lua; @@ -28,7 +29,7 @@ pub struct Config { /// It stores the option to whether enable or disable debug mode. pub debug: bool, /// It stores all the engine names that were enabled by the user. - pub upstream_search_engines: Vec, + pub upstream_search_engines: Vec, /// It stores the time (secs) which controls the server request timeout. pub request_timeout: u8, /// It stores the number of threads which controls the app will use to run. @@ -111,8 +112,8 @@ impl Config { .get::<_, HashMap>("upstream_search_engines")? .into_iter() .filter_map(|(key, value)| value.then_some(key)) - .filter_map(|engine| crate::models::engine_models::EngineHandler::new(&engine).ok()) - .collect(), + .map(|engine| EngineHandler::new(&engine)) + .collect::, error_stack::Report>>()?, request_timeout: globals.get::<_, u8>("request_timeout")?, threads, rate_limiter: RateLimiter { From f56002dca67a0213981f3948b850d3dba8343d69 Mon Sep 17 00:00:00 2001 From: Zsombor Gegesy Date: Sun, 8 Oct 2023 22:30:31 +0200 Subject: [PATCH 23/25] Rename the error to NoSuchEngineFound and add the name of missing engine to it --- src/models/aggregation_models.rs | 4 ++-- src/models/engine_models.rs | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/models/aggregation_models.rs b/src/models/aggregation_models.rs index 6e4bddf..660804e 100644 --- a/src/models/aggregation_models.rs +++ b/src/models/aggregation_models.rs @@ -85,14 +85,14 @@ impl EngineErrorInfo { pub fn new(error: &EngineError, engine: &str) -> Self { Self { error: match error { - EngineError::EngineNotFound => "EngineNotFound".to_owned(), + EngineError::NoSuchEngineFound(_) => "EngineNotFound".to_owned(), EngineError::RequestError => "RequestError".to_owned(), EngineError::EmptyResultSet => "EmptyResultSet".to_owned(), EngineError::UnexpectedError => "UnexpectedError".to_owned(), }, engine: engine.to_owned(), severity_color: match error { - EngineError::EngineNotFound => "red".to_owned(), + EngineError::NoSuchEngineFound(_) => "red".to_owned(), EngineError::RequestError => "green".to_owned(), EngineError::EmptyResultSet => "blue".to_owned(), EngineError::UnexpectedError => "red".to_owned(), diff --git a/src/models/engine_models.rs b/src/models/engine_models.rs index d53fc0c..05b5a11 100644 --- a/src/models/engine_models.rs +++ b/src/models/engine_models.rs @@ -9,7 +9,7 @@ use std::{collections::HashMap, fmt, time::Duration}; #[derive(Debug)] pub enum EngineError { /// No matching engine found - EngineNotFound, + NoSuchEngineFound(String), /// This variant handles all request related errors like forbidden, not found, /// etc. EmptyResultSet, @@ -26,8 +26,8 @@ pub enum EngineError { impl fmt::Display for EngineError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - EngineError::EngineNotFound => { - write!(f, "Search engine not found") + EngineError::NoSuchEngineFound(engine) => { + write!(f, "No such engine with the name '{engine}' found") } EngineError::EmptyResultSet => { write!(f, "The upstream search engine returned an empty result set") @@ -150,7 +150,11 @@ impl EngineHandler { let engine = crate::engines::searx::Searx::new()?; ("searx", Box::new(engine)) } - _ => return Err(Report::from(EngineError::EngineNotFound)), + _ => { + return Err(Report::from(EngineError::NoSuchEngineFound( + engine_name.to_string(), + ))) + } }; Ok(Self { From 8156f7ea79099d3c64739bf94840db14ab134aad Mon Sep 17 00:00:00 2001 From: Ananth B Prathap <136650032+KekmaTime@users.noreply.github.com> Date: Mon, 9 Oct 2023 10:09:12 -0700 Subject: [PATCH 24/25] =?UTF-8?q?=F0=9F=94=A7=20Remove=20the=20unused=20ma?= =?UTF-8?q?pped=20ports=20for=20docker=20deployment=20(#309)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * removed-ports * 🚀 chore: bump the app version (#309) * 🚀 chore: bump the app version (#309) --------- Co-authored-by: alamin655 --- Cargo.lock | 2 +- Cargo.toml | 2 +- docker-compose.yml | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 560acdb..2074fd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3965,7 +3965,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "1.0.9" +version = "1.0.11" dependencies = [ "actix-cors", "actix-files", diff --git a/Cargo.toml b/Cargo.toml index 2e543b6..2b12cac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "1.0.9" +version = "1.0.11" edition = "2021" description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind." repository = "https://github.com/neon-mmd/websurfx" diff --git a/docker-compose.yml b/docker-compose.yml index 6b50b24..1bfa54e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,4 @@ services: - ./websurfx/:/etc/xdg/websurfx/ # Uncomment the following lines if you are using the `hybrid` or `redis` caching feature. # redis: - # image: redis:latest - # ports: - # - 6379:6379 \ No newline at end of file + # image: redis:latest \ No newline at end of file From a912ac0724facb7ec871a06491d1a770b660b3c0 Mon Sep 17 00:00:00 2001 From: Ajeet Singh <34164838+Ajeets6@users.noreply.github.com> Date: Fri, 13 Oct 2023 21:00:38 +0530 Subject: [PATCH 25/25] =?UTF-8?q?=F0=9F=93=9D=20Remove=20the=20two=20unnec?= =?UTF-8?q?essary=20lines=20from=20the=20code=20snippet=20(#312)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Removed unnecessary lines(issue:310) - # ports: - # - 6379:6379 * Update docs/installation.md Co-authored-by: neon_arch --------- Co-authored-by: neon_arch Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- docs/installation.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 1c4b112..8544dab 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -217,8 +217,6 @@ services: # Uncomment the following lines if you are using the `hybrid/latest` or `redis` image. # redis: # image: redis:latest - # ports: - # - 6379:6379 ``` Then make sure to edit the `docker-compose.yml` file as required. After that create a directory `websurfx` in the directory you have placed the `docker-compose.yml` file, and then in the new directory create two new empty files named `allowlist.txt` and `blocklist.txt`. Finally, create a new config file `config.lua` with the default configuration, which looks something like this: