diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml new file mode 100644 index 0000000..fcb0277 --- /dev/null +++ b/.github/workflows/contributors.yml @@ -0,0 +1,47 @@ +name: Contributors List + +on: + workflow_dispatch: + + schedule: + - cron: "0 1 * * *" + +jobs: + contributors: + permissions: + contents: write + pull-requests: write + + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + with: + fetch-depth: 0 + ref: ${{ github.event.repository.default_branch }} + + - name: Update contributors list + uses: wow-actions/contributors-list@b9e91f91a51a55460fdcae64daad0cb8122cdd53 # v1.1.0 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + svgPath: images/contributors_list.svg + round: true + includeBots: false + noCommit: true + + - name: Commit & PR + uses: peter-evans/create-pull-request@38e0b6e68b4c852a5500a94740f0e535e0d7ba54 # v4.2.4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + add-paths: .github/assets/CONTRIBUTORS.svg + commit-message: 'chore: update contributors-list' + committer: GitHub + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: false + branch: workflow/update-contributors-list + base: main + delete-branch: true + title: 'chore: update contributors-list' + body: | + Automated update to `images/contributors_list.svg` \ No newline at end of file diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index c8adbf7..9dd3ea9 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -3,7 +3,7 @@ name: Import open source standard labels on: push: branches: - - master + - rolling jobs: labels: diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml deleted file mode 100644 index 8e1e46b..0000000 --- a/.github/workflows/releases.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Releases -on: - push: - branches: - - "rolling" - -concurrency: - group: "rolling-branch" - -jobs: - changelog: - if: github.repository == 'neon-mmd/websurfx' - runs-on: ubuntu-latest - - steps: - # Create a temporary, uniquely named branch to push release info to - - name: create temporary branch - uses: peterjgrainger/action-create-branch@v2.3.0 - id: create-branch - with: - branch: "release-from-${{ github.sha }}" - sha: "${{ github.sha }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # check out the repository afterwards - - uses: actions/checkout@v3 - - # fetch branches and switch to the temporary branch - - name: switch to new branch - run: git fetch --all && git checkout --track origin/release-from-${{ github.sha }} - - # update app config with version - - name: Get current rust app version from its Cargo.toml. - id: foo - uses: dante-signal31/rust-app-version@v1.2.0 - with: - cargo_toml_folder: rust_app_folder/ - - - name: Use the version to update the Cargo.toml version. - shell: bash - run: sed -i "3s/version = \"[0-9]*.[0-9]*.[0-9]*\"/version = \"${{ steps.foo.outputs.app_version }}\"/g" Cargo.toml - - # create release info and push it upstream - - name: conventional Changelog Action - id: changelog - uses: TriPSs/conventional-changelog-action@v3 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - version-file: "./Cargo.toml" - git-branch: "release-from-${{ github.sha }}" - skip-on-empty: false - skip-git-pull: true - - # create PR using GitHub CLI - - name: create PR with release info - id: create-pr - run: gh pr create --base main --head release-from-${{ github.sha }} --title 'Merge new release into rolling' --body 'Created by Github action' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # merge PR using GitHub CLI - - name: merge PR with release info - id: merge-pr - run: gh pr merge --admin --merge --subject 'Merge release info' --delete-branch - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # release info is now in main so we can continue as before - - name: create release with last commit - uses: actions/create-release@v1 - if: steps.changelog.outputs.skipped == 'false' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ steps.changelog.outputs.tag }} - release_name: ${{ steps.changelog.outputs.tag }} - body: ${{ steps.changelog.outputs.clean_changelog }} diff --git a/.github/workflows/rust_format.yml b/.github/workflows/rust_format.yml index d865c8c..c300863 100644 --- a/.github/workflows/rust_format.yml +++ b/.github/workflows/rust_format.yml @@ -19,7 +19,16 @@ jobs: profile: minimal toolchain: stable components: rustfmt, clippy - + - name: Format + uses: actions-rs/cargo@v1 + with: + command: fmt + args: -- --check + - name: Clippy + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --all-features --all-targets --all - name: Run cargo check uses: actions-rs/cargo@v1 with: diff --git a/.gitignore b/.gitignore index ea8c4bf..5e79d6b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ /target +package.json +package-lock.json +dump.rdb +.vscode diff --git a/.rusty-hook.toml b/.rusty-hook.toml new file mode 100644 index 0000000..105b318 --- /dev/null +++ b/.rusty-hook.toml @@ -0,0 +1,5 @@ +[hooks] +pre-commit = "cargo test && cargo fmt -- --check && cargo clippy && stylelint ./public/static/themes/*.css ./public/static/colorschemes/*.css ./public/static/*.js" + +[logging] +verbose = true diff --git a/.stylelintrc.json b/.stylelintrc.json new file mode 100644 index 0000000..9019f4f --- /dev/null +++ b/.stylelintrc.json @@ -0,0 +1,13 @@ +{ + "extends": "stylelint-config-standard", + "rules": { + "alpha-value-notation": "number", + "selector-class-pattern": null + }, +"overrides": [ + { + "files": ["*.js"], + "customSyntax": "postcss-lit" + } + ] +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d630e2a..324dfbf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ ## Documentation/Wiki -Found a typo, or something that isn't as clear as it could be? Maybe I've missed something off altogether, or you hit a roadblock that took you a while to figure out. Edit the [wiki](https://github.com/neon-mmd/websurfx/wiki) to add to or improve the documentation. This will help future users get Websurfx up and running more easily. +Found a typo, or something that isn't as clear as it could be? Maybe I've missed something off altogether, or you hit a roadblock that took you a while to figure out. Edit the [docs](./docs/) to add to or improve the documentation. This will help future users get Websurfx up and running more easily. ## Readme @@ -48,23 +48,7 @@ We have a [Discord](https://discord.gg/SWnda7Mw5u) channel, feel free to join an # Where To Contribute? -## For Source Code Contributions - -The _rolling branch_ is where we intend all source code contributions should go. - -## For Readme Contributions - -The _master branch_ is where we intend all source code contributions should go. - -# How To Fork - -![image](./images/fork_button.png) - -![image](./images/fork_options_page.png) - -Please make sure to leave the `Copy the master branch only` option ticked off. - -![image](./images/create_fork_button.png) +The _rolling branch_ is where we intend all contributions should go. We appreciate any contributions whether be of any size or topic and suggestions to help improve the Websurfx project. Please keep in mind the above requirements and guidelines before submitting a pull request and also if you have any doubts/concerns/questions about the project, its source code or anything related to the project than feel free to ask by opening an [issue](https://github.com/neon-mmd/websurfx/issues) or by asking us on our [Discord](https://discord.gg/SWnda7Mw5u) channel. diff --git a/Cargo.lock b/Cargo.lock index 85310e9..c3b4f93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,19 +4,19 @@ version = 3 [[package]] name = "actix-codec" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" +checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ "bitflags", "bytes 1.4.0", "futures-core", "futures-sink", - "log", "memchr", "pin-project-lite", - "tokio 1.28.0", + "tokio 1.28.2", "tokio-util", + "tracing", ] [[package]] @@ -53,7 +53,7 @@ dependencies = [ "actix-service", "actix-utils", "ahash 0.8.3", - "base64 0.21.0", + "base64 0.21.2", "bitflags", "brotli", "bytes 1.4.0", @@ -62,7 +62,7 @@ dependencies = [ "encoding_rs", "flate2", "futures-core", - "h2 0.3.18", + "h2 0.3.19", "http 0.2.9", "httparse", "httpdate", @@ -75,7 +75,7 @@ dependencies = [ "rand 0.8.5", "sha1", "smallvec 1.10.0", - "tokio 1.28.0", + "tokio 1.28.2", "tokio-util", "tracing", "zstd", @@ -87,7 +87,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "syn 1.0.109", ] @@ -111,7 +111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" dependencies = [ "futures-core", - "tokio 1.28.0", + "tokio 1.28.2", ] [[package]] @@ -125,10 +125,10 @@ dependencies = [ "actix-utils", "futures-core", "futures-util", - "mio 0.8.6", + "mio 0.8.8", "num_cpus", "socket2", - "tokio 1.28.0", + "tokio 1.28.2", "tracing", ] @@ -201,8 +201,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" dependencies = [ "actix-router", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -315,9 +315,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "bit-set" @@ -381,9 +381,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -438,6 +438,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "ci_info" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f638c70e8c5753795cc9a8c07c44da91554a09e4cf11a7326e8161b0a3c45e" +dependencies = [ + "envmnt", +] + [[package]] name = "cloudabi" version = "0.0.3" @@ -605,8 +614,8 @@ dependencies = [ "itoa 1.0.6", "matches", "phf 0.10.1", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "smallvec 1.10.0", "syn 1.0.109", ] @@ -617,7 +626,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "syn 1.0.109", ] @@ -628,17 +637,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "rustc_version 0.4.0", "syn 1.0.109", ] [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -693,6 +702,16 @@ dependencies = [ "termcolor", ] +[[package]] +name = "envmnt" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d328fc287c61314c4a61af7cfdcbd7e678e39778488c7cb13ec133ce0f4059" +dependencies = [ + "fsio", + "indexmap", +] + [[package]] name = "errno" version = "0.3.1" @@ -730,8 +749,8 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", "synstructure", ] @@ -796,6 +815,12 @@ dependencies = [ "percent-encoding 2.2.0", ] +[[package]] +name = "fsio" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1fd087255f739f4f1aeea69f11b72f8080e9c2e7645cd06955dad4a178a49e3" + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -959,9 +984,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ "bytes 1.4.0", "fnv", @@ -971,16 +996,16 @@ dependencies = [ "http 0.2.9", "indexmap", "slab", - "tokio 1.28.0", + "tokio 1.28.2", "tokio-util", "tracing", ] [[package]] name = "handlebars" -version = "4.3.6" +version = "4.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "035ef95d03713f2c347a72547b7cd38cbc9af7cd51e6099fb62d586d4a6dee3a" +checksum = "83c3372087601b532857d332f5957cbae686da52bb7810bf038c3e3c3cc2fa0d" dependencies = [ "log", "pest", @@ -1035,8 +1060,8 @@ dependencies = [ "log", "mac", "markup5ever 0.11.0", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -1149,7 +1174,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.18", + "h2 0.3.19", "http 0.2.9", "http-body 0.4.5", "httparse", @@ -1157,7 +1182,7 @@ dependencies = [ "itoa 1.0.6", "pin-project-lite", "socket2", - "tokio 1.28.0", + "tokio 1.28.2", "tower-service", "tracing", "want 0.3.0", @@ -1185,7 +1210,7 @@ dependencies = [ "bytes 1.4.0", "hyper 0.14.26", "native-tls", - "tokio 1.28.0", + "tokio 1.28.2", "tokio-native-tls", ] @@ -1242,9 +1267,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -1301,9 +1326,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.62" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68c16e1bfd491478ab155fd8b4896b86f9ede344949b641e61501e07c2b8b4d5" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] @@ -1338,9 +1363,9 @@ checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "linux-raw-sys" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "local-channel" @@ -1381,12 +1406,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" [[package]] name = "mac" @@ -1513,14 +1535,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -1570,6 +1592,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" +[[package]] +name = "nias" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab250442c86f1850815b5d268639dff018c0627022bc1940eb2d642ca1ce12f0" + [[package]] name = "nodrop" version = "0.1.14" @@ -1606,15 +1634,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" [[package]] name = "openssl" -version = "0.10.52" +version = "0.10.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" +checksum = "12df40a956736488b7b44fe79fe12d4f245bb5b3f5a1f6095e499760015be392" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -1631,9 +1659,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -1644,9 +1672,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.87" +version = "0.9.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" +checksum = "c2ce0f250f34a308dcfdbb351f511359857d4ed2134ba715a4eadd46e1ffd617" dependencies = [ "cc", "libc", @@ -1749,9 +1777,9 @@ checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -1863,8 +1891,8 @@ dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", "proc-macro-hack", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", ] @@ -1942,9 +1970,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" dependencies = [ "unicode-ident", ] @@ -1970,11 +1998,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ - "proc-macro2 1.0.56", + "proc-macro2 1.0.59", ] [[package]] @@ -2213,9 +2241,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.1" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" dependencies = [ "aho-corasick", "memchr", @@ -2224,9 +2252,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "reqwest" @@ -2264,16 +2292,16 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.17" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", "bytes 1.4.0", "encoding_rs", "futures-core", "futures-util", - "h2 0.3.18", + "h2 0.3.19", "http 0.2.9", "http-body 0.4.5", "hyper 0.14.26", @@ -2289,7 +2317,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.1", - "tokio 1.28.0", + "tokio 1.28.2", "tokio-native-tls", "tower-service", "url 2.3.1", @@ -2361,6 +2389,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rusty-hook" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96cee9be61be7e1cbadd851e58ed7449c29c620f00b23df937cb9cbc04ac21a3" +dependencies = [ + "ci_info", + "getopts", + "nias", + "toml", +] + [[package]] name = "ryu" version = "1.0.13" @@ -2410,9 +2450,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ "bitflags", "core-foundation", @@ -2423,9 +2463,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" dependencies = [ "core-foundation-sys", "libc", @@ -2482,22 +2522,22 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.162" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b2f6e1ab5c2b98c05f0f35b236b22e8df7ead6ffbf51d7808da7f8817e7ab6" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.162" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2a0814352fd64b58489904a44ea8d90cb1a91dcb6b4f5ebabc32c8318e93cb6" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2680,8 +2720,8 @@ checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" dependencies = [ "phf_generator 0.7.24", "phf_shared 0.7.24", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "string_cache_shared", ] @@ -2693,8 +2733,8 @@ checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" dependencies = [ "phf_generator 0.10.0", "phf_shared 0.10.0", - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", ] [[package]] @@ -2720,19 +2760,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.15" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "unicode-ident", ] @@ -2742,8 +2782,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", + "proc-macro2 1.0.59", + "quote 1.0.28", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -2796,9 +2836,9 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2875,14 +2915,14 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg 1.1.0", "bytes 1.4.0", "libc", - "mio 0.8.6", + "mio 0.8.8", "num_cpus", "parking_lot 0.12.1", "pin-project-lite", @@ -2940,9 +2980,9 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", ] [[package]] @@ -2952,7 +2992,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.28.0", + "tokio 1.28.2", ] [[package]] @@ -3037,10 +3077,19 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.28.0", + "tokio 1.28.2", "tracing", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -3061,9 +3110,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", ] @@ -3112,9 +3161,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-normalization" @@ -3243,9 +3292,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b6cb788c4e39112fbe1822277ef6fb3c55cd86b95cb3d3c4c1c9597e4ac74b4" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -3253,24 +3302,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e522ed4105a9d626d885b35d62501b30d9666283a5c8be12c14a8bdafe7822" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.35" +version = "0.4.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "083abe15c5d88556b77bdf7aef403625be9e327ad37c62c4e4129af740168163" +checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3280,38 +3329,38 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "358a79a0cb89d21db8120cbfb91392335913e4890665b1a7981d9e956903b434" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ - "quote 1.0.27", + "quote 1.0.28", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4783ce29f09b9d93134d41297aded3a712b7b979e9c6f28c32cb88c973a94869" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ - "proc-macro2 1.0.56", - "quote 1.0.27", - "syn 2.0.15", + "proc-macro2 1.0.59", + "quote 1.0.28", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.85" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a901d592cafaa4d711bc324edfaff879ac700b19c3dfd60058d2b445be2691eb" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "web-sys" -version = "0.3.62" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b5f940c7edfdc6d12126d98c9ef4d1b3d470011c47c76a6581df47ad9ba721" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" dependencies = [ "js-sys", "wasm-bindgen", @@ -3319,7 +3368,7 @@ dependencies = [ [[package]] name = "websurfx" -version = "0.6.0" +version = "0.12.0" dependencies = [ "actix-files", "actix-web", @@ -3328,14 +3377,16 @@ dependencies = [ "handlebars", "log", "md5", + "once_cell", "rand 0.8.5", "redis", - "reqwest 0.11.17", + "reqwest 0.11.18", "rlua", + "rusty-hook", "scraper", "serde", "serde_json", - "tokio 1.28.0", + "tokio 1.28.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ce99ca3..b8f2755 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websurfx" -version = "0.6.0" +version = "0.12.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -21,3 +21,7 @@ rlua = {version="*"} redis = {version="*"} md5 = {version="*"} rand={version="*"} +once_cell = {version="*"} + +[dev-dependencies] +rusty-hook = "^0.11.2" diff --git a/README.md b/README.md index 1ea56f7..36b358b 100644 --- a/README.md +++ b/README.md @@ -5,68 +5,26 @@ Readme | Discord | GitHub | - Documentation + Documentation

- - GitHub - - - GitHub Repo stars - - - GitHub forks - GitHub code size in bytes - - GitHub issues - - - GitHub pull requests - GitHub Workflow Status - - GitHub release (latest by date including pre-releases) - Maintenance - - GitHub contributors - Gitpod -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # Features 🚀 @@ -136,20 +94,23 @@ - 💨 Ad-free and clean results - 🌟 and lots more... -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # Installation and Testing 🛠️ -> For full setup instructions, see: [**Installation**](https://github.com/neon-mmd/websurfx/wiki/installation) +> For full setup instructions, see: [**Installation**](./docs/installation.md) -To get started with Websurfx, clone the repository, edit the config file, which is located in the `websurfx`{.verbatim} directory, and install the Redis server by following the instructions located [here](https://redis.io/docs/getting-started/) and then run the websurfx server and redis server using the following commands: +Before you can start building `websurfx`, you will need to have `Cargo` installed on your system. You can find the installation instructions [here](https://doc.rust-lang.org/cargo/getting-started/installation.html). + +To get started with Websurfx, clone the repository, edit the config file, which is located in the `websurfx/` directory, and install the Redis server by following the instructions located [here](https://redis.io/docs/getting-started/) and then run the websurfx server and redis server using the following commands: ``` shell git clone https://github.com/neon-mmd/websurfx.git cd websurfx -cargo build +git checkout stable +cargo build -r redis-server --port 8082 & -./target/debug/websurfx +./target/release/websurfx ``` Once you have started the server, open your preferred web browser and navigate to to start using Websurfx. @@ -157,42 +118,42 @@ Once you have started the server, open your preferred web browser and navigate t > **Warning** > Please be aware that the project is still in the testing phase and is not ready for production use. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # Configuration 🔧 -> For full configuration instructions, see: [**Configuration**](https://github.com/neon-mmd/websurfx/wiki/configuration) +> For full configuration instructions, see: [**Configuration**](./docs/configuration.md) Websurfx is configured through the config.lua file, located at `websurfx/config.lua`. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # Theming 🎨 -> For full theming and customization instructions, see: [**Theming**](https://github.com/neon-mmd/websurfx/wiki/theming) +> For full theming and customization instructions, see: [**Theming**](./docs/theming.md) Websurfx comes with several themes and color schemes by default, which you can apply and edit through the config file. Supports custom themes and color schemes using CSS, allowing you to develop your own unique-looking website. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # Multi-Language Support 🌍 > **Note** > Currently, we do not support other languages, but in the future, we will start accepting contributions regarding language support because we believe that language should not be a barrier to entry. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # System Requirements 📊 At present, we only support x86_64 architecture systems, but we would love to have contributions that extend to other architectures as well. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # FAQ (Frequently Asked Questions) 🗨️ ## Why Websurfx? -The primary purpose of the Websurfx project is to create a fast, secure, and privacy-focused [meta-search engine](https://en.wikipedia.org/wiki/Metasearch_engine). While there are numerous meta-search engines available, not all of them guarantee the security of their search engine, which is critical for maintaining privacy. Memory flaws, for example, can expose private or sensitive information, which is never a good thing. Also, there is the added problem of Spam, ads, and unorganic results which most engines don't have the full-proof answer to it till now but with Websurfx I finally put a full stop to this problem, also, Rust is used to write Websurfx, which ensures memory safety and removes such issues. Many meta-search engines also lack important features like advanced picture search, which is required by many graphic designers, content providers, and others. Websurfx attempts to improve the user experience by providing these and other features, such as proper NSFW blocking and Micro-apps or Quick results (like providing a calculator, currency exchanges, etc in the search results). +The primary purpose of the Websurfx project is to create a fast, secure, and privacy-focused meta-search engine. While there are numerous meta-search engines available, not all of them guarantee the security of their search engine, which is critical for maintaining privacy. Memory flaws, for example, can expose private or sensitive information, which is never a good thing. Also, there is the added problem of Spam, ads, and unorganic results which most engines don't have the full-proof answer to it till now but with Websurfx I finally put a full stop to this problem, also, Rust is used to write Websurfx, which ensures memory safety and removes such issues. Many meta-search engines also lack important features like advanced picture search, which is required by many graphic designers, content providers, and others. Websurfx attempts to improve the user experience by providing these and other features, such as proper NSFW blocking and Micro-apps or Quick results (like providing a calculator, currency exchanges, etc in the search results). ## Why AGPLv3? @@ -202,13 +163,13 @@ Websurfx is distributed under the **AGPLv3** license to keep the source code ope Rust was chosen as the programming language for Websurfx because of its memory safety features, which can help prevent vulnerabilities and make the codebase more secure. Rust is also faster than C++, which contributes to Websurfx's speed and responsiveness. Furthermore, the Rust ownership and borrowing system enables secure concurrency and thread safety in the program. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # More Contributers Wanted 📣 We are looking for more willing contributors to help grow this project. For more information on how you can contribute, check out the [project board](https://github.com/neon-mmd/websurfx/projects?query=is%3Aopen) and the [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines and rules for making contributions. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # Supporting Websurfx 💖 @@ -224,20 +185,20 @@ Several areas that we need a bit of help with at the moment are: - Submit a PR to add a new feature, fix a bug, update the docs, add a theme, widget, or something else. - Star Websurfx on GitHub. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # Documentation 📘 > **Note** -> We welcome any contributions to the [documentation](https://github.com/neon-mmd/websurfx/wiki) as this will benefit everyone who uses this project. +> We welcome any contributions to the [documentation](./docs/) as this will benefit everyone who uses this project. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # Roadmap 🛣️ > Coming soon!! 🙂. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # Contributing 🙋 @@ -251,13 +212,13 @@ Check out this [video](https://youtu.be/FccdqCucVSI) by Mr. Nick on how to contr If you are a developer, have a look at the [CONTRIBUTING.org](CONTRIBUTING.md) document for more information. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # License 📜 Websurfx is licensed under the [AGPLv3](LICENSE) license. -**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** # Credits 🤝 @@ -271,7 +232,7 @@ We would like to thank the following people for their contributions and support:

-**[⬆️ Back to Top](#websurfx)** +**[⬆️ Back to Top](#--)** --- diff --git a/docs/configuration.md b/docs/configuration.md index be0325a..0d54fd9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,17 +1,33 @@ # Configuration -Everything in websurfx can be configured through the config file located at `websurfx/config.lua`. +## Installed From Source -Some of the configuration options provided in the file are stated below. These are subdivided into three categories: +If you have built `websurfx` from source then the configuration file will be located under project directory (codebase) at `websurfx/` +> **Note** +> If you have built websurfx with unstable/rolling/edge branch then you can copy the configuration file from `websurfx/config.lua` located under project directory (codebase) to `~/.config/websurfx/` and make the changes there and rerun the websurfx server. _This is only available from unstable/rolling/edge version_. + +## Installed From Package + +If you have installed `websurfx` using the package manager of your Linux distro then the default configuration file will be located at `/etc/xdg/websurfx/`. You can copy the default config to `~/.config/websurfx/` and make the changes there and rerun the websurfx server. + +Some of the configuration options provided in the file are stated below. These are subdivided into the following categories: + +- General - Server - Website - Cache +# General + +- **logging:** An option to enable or disable logs. +- **debug:** An option to enable or disable debug mode. + ## Server - **port:** Port number on which server should be launched. - **binding_ip_addr:** IP address on the which server should be launched. +- **production_use:** Whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users). If production_use is set to true. There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. This is newly added option and hence is only available in the **edge version**. ## Website @@ -39,4 +55,4 @@ Some of the configuration options provided in the file are stated below. These a - **redis_connection_url:** Redis connection url address on which the client should connect on. -[⬅️ Go back to Home](https://github.com/neon-mmd/websurfx/wiki). +[⬅️ Go back to Home](./README.md) diff --git a/docs/faq.md b/docs/faq.md index 1805487..cbd3582 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -2,7 +2,7 @@ ## Why Websurfx? -The main goal of the Websurfx project is to provide a fast, secure, and privacy-focused [meta search engine](https://en.wikipedia.org/wiki/Metasearch_engine). While there are many meta search engines available, they do not always guarantee the security of their search engine, which is essential for ensuring privacy. For example, memory vulnerabilities can leak private or sensitive information, which is never good. Websurfx is written in Rust, which guarantees memory safety and eliminates such problems. Many meta search engines also lack key features such as advanced image search, which is required by many graphic designers, content creators, and others. Websurfx aims to provide these features and others, such as proper NSFW blocking, to improve the user experience. +The primary purpose of the Websurfx project is to create a fast, secure, and privacy-focused [meta-search engine](https://en.wikipedia.org/wiki/Metasearch_engine). While there are numerous meta-search engines available, not all of them guarantee the security of their search engine, which is critical for maintaining privacy. Memory flaws, for example, can expose private or sensitive information, which is never a good thing. Also, there is the added problem of Spam, ads, and unorganic results which most engines don't have the full-proof answer to it till now but with Websurfx I finally put a full stop to this problem, also, Rust is used to write Websurfx, which ensures memory safety and removes such issues. Many meta-search engines also lack important features like advanced picture search, which is required by many graphic designers, content providers, and others. Websurfx attempts to improve the user experience by providing these and other features, such as proper NSFW blocking and Micro-apps or Quick results (like providing a calculator, currency exchanges, etc in the search results). ## Why AGPLv3? @@ -12,4 +12,4 @@ Websurfx is released under the AGPLv3 license to ensure that the source code rem Rust was chosen as the programming language for Websurfx due to its memory safety features, which can help prevent vulnerabilities and make the codebase more secure. Rust is also faster than C++, which helps to make Websurfx fast and responsive. In addition, Rust's ownership and borrowing system allows for safe concurrency and thread safety in the codebase. -[⬅️ Go back to Home](https://github.com/neon-mmd/websurfx/wiki). +[⬅️ Go back to Home](./README.md) diff --git a/docs/installation.md b/docs/installation.md index c9aa6f1..fbb0d16 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,35 +1,70 @@ -# Stable +# Install From Package -To get started with Websurfx, clone the repository, edit the config file which is located in the `websurfx` directory and install redis server by following the instructions located [here](https://redis.io/docs/getting-started/) and then run the websurfx server and redis server using the following commands: +## Arch Linux + +You can install `Websurfx` through the [Aur](https://aur.archlinux.org/packages/websurfx-git), Currently we only support `Rolling/Edge` version. You can install the rolling/edge version by running the following command (using [paru](https://github.com/Morganamilo/paru)): + +```bash +paru -S websurfx-edge-git +``` + +After installing it you can run the websurfx server by running the following commands: + +``` bash +redis-server --port 8082 & +websurfx +``` + +Once you have started the server, open your preferred web browser and navigate to http://127.0.0.1:8080/ to start using Websurfx. + +If you want to change the port or the ip or any other configuration setting checkout the [configuration docs](./configuration.md). + +## Other Distros + +The package is currently not available on other Linux distros. With contribution and support it can be made available on other distros as well 🙂. + +# Install From Source + +Before you can start building `websurfx`, you will need to have `Cargo` installed on your system. You can find the installation instructions [here](https://doc.rust-lang.org/cargo/getting-started/installation.html). + +## Stable + +To get started with Websurfx, clone the repository, edit the config file which is located in the `websurfx` directory and install redis server by following the instructions located [here](https://redis.io/docs/getting-started/) and then build and run the websurfx server by running the following commands: ```shell git clone https://github.com/neon-mmd/websurfx.git cd websurfx -cargo build +git checkout stable +cargo build -r redis-server --port 8082 & -./target/debug/websurfx +./target/release/websurfx ``` -# Rolling/Edge/Unstable +Once you have started the server, open your preferred web browser and navigate to http://127.0.0.1:8080/ to start using Websurfx. + +If you want to change the port or the ip or any other configuration setting checkout the [configuration docs](./configuration.md). + +## Rolling/Edge/Unstable If you want to use the rolling/edge branch, run the following commands instead: ```shell git clone https://github.com/neon-mmd/websurfx.git cd websurfx -git checkout rolling -cargo build +cargo build -r redis-server --port 8082 & -./target/debug/websurfx +./target/release/websurfx ``` Once you have started the server, open your preferred web browser and navigate to http://127.0.0.1:8080/ to start using Websurfx. +If you want to change the port or the ip or any other configuration setting checkout the [configuration docs](./configuration.md). + # Docker Deployment Before you start, you will need [Docker](https://docs.docker.com/get-docker/) installed on your system first. -## Stable +## Unstable/Edge/Rolling First clone the the repository by running the following command: @@ -41,11 +76,18 @@ cd websurfx After that edit the config.lua file located under `websurfx` directory. In the config file you will specifically need to change to values which is `binding_ip_addr` and `redis_connection_url` which should make the config look something like this: ```lua --- Server +-- ### General ### +logging = true -- an option to enable or disable logs. +debug = false -- an option to enable or disable debug mode. + +-- ### Server ### port = "8080" -- port on which server should be launched binding_ip_addr = "0.0.0.0" --ip address on the which server should be launched. +production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) +-- if production_use is set to true +-- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. --- Website +-- ### Website ### -- The different colorschemes provided are: -- {{ -- catppuccin-mocha @@ -60,7 +102,7 @@ binding_ip_addr = "0.0.0.0" --ip address on the which server should be launched. colorscheme = "catppuccin-mocha" -- the colorscheme name which should be used for the website theme theme = "simple" -- the theme name which should be used for the website --- Caching +-- ### Caching ### redis_connection_url = "redis://redis:6379" -- redis connection url address on which the client should connect on. ``` @@ -72,14 +114,14 @@ docker compose up -d --build This will take around 5-10 mins for first deployment, afterwards the docker build stages will be cached so it will be faster to be build from next time onwards. After the above step finishes launch your preferred browser and then navigate to `http://:`. -## Unstable/Edge/Rolling +## Stable -For the unstable/rolling/edge version, follow the same steps as above with an addition of one command for cloning the repository which makes the cloning step as follows: +For the stable version, follow the same steps as above (as mentioned for the unstable/rolling/edge version) with an addition of one command which has to be performed after cloning and changing directory into the repository which makes the cloning step as follows: ```bash git clone https://github.com/neon-mmd/websurfx.git cd websurfx -git checkout rolling +git checkout stable ``` -[⬅️ Go back to Home](https://github.com/neon-mmd/websurfx/wiki). +[⬅️ Go back to Home](./README.md) diff --git a/docs/introduction.md b/docs/introduction.md index 476f509..7b540fb 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -10,4 +10,4 @@ Most meta search engines tend to be slow, lack high level of customization and m Websurfx is a project which seeks to provide privacy, security, speed and all the features which the user wants. -[⬅️ Go back to Home](https://github.com/neon-mmd/websurfx/wiki). +[⬅️ Go back to Home](./README.md) diff --git a/docs/theming.md b/docs/theming.md index 95ff7ff..49f3f22 100644 --- a/docs/theming.md +++ b/docs/theming.md @@ -321,4 +321,45 @@ footer { } ``` -[⬅️ Go back to Home](https://github.com/neon-mmd/websurfx/wiki). +### Styles for the about page + +This part is only available right now in the **rolling/edge/unstable** version + +```css +.about-container article{ + font-size: 1.5rem; + color:var(--fg); + padding-bottom: 10px; + } + +.about-container article h1{ + color: var(--2); + font-size: 2.8rem; + } + +.about-container article div{ + padding-bottom: 15px; + } + +.about-container a{ + color:var(--3); +} + +.about-container article h2{ + color: var(--3); + font-size: 1.8rem; + padding-bottom: 10px; +} + +.about-container p{ + color:var(--fg); + font-size: 1.6rem; + padding-bottom: 10px; +} + +.about-container h3{ + font-size: 1.5rem; +} +``` + +[⬅️ Go back to Home](./README.md) diff --git a/images/create_fork_button.png b/images/create_fork_button.png deleted file mode 100644 index c060fd4..0000000 Binary files a/images/create_fork_button.png and /dev/null differ diff --git a/images/fork_button.png b/images/fork_button.png deleted file mode 100644 index f86ae80..0000000 Binary files a/images/fork_button.png and /dev/null differ diff --git a/images/fork_options_page.png b/images/fork_options_page.png deleted file mode 100644 index f8e9c24..0000000 Binary files a/images/fork_options_page.png and /dev/null differ diff --git a/images/main_page.png b/images/main_page.png index 0db1222..337eac5 100644 Binary files a/images/main_page.png and b/images/main_page.png differ diff --git a/public/images/websurfx_logo.png b/public/images/websurfx_logo.png new file mode 100644 index 0000000..9449e33 Binary files /dev/null and b/public/images/websurfx_logo.png differ diff --git a/public/static/colorschemes/dark-chocolate.css b/public/static/colorschemes/dark-chocolate.css new file mode 100644 index 0000000..f1d6848 --- /dev/null +++ b/public/static/colorschemes/dark-chocolate.css @@ -0,0 +1,11 @@ +:root { + --bg: #000000; + --fg: #ffffff; + --1: #121212; + --2: #808080; + --3: #999999; + --4: #666666; + --5: #bfbfbf; + --6: #e0e0e0; + --7: #555555; + } diff --git a/public/static/colorschemes/one-dark.css b/public/static/colorschemes/one-dark.css new file mode 100644 index 0000000..0afb05e --- /dev/null +++ b/public/static/colorschemes/one-dark.css @@ -0,0 +1,11 @@ +:root { + --bg: #282c34; + --fg: #abb2bf; + --1: #3b4048; + --2: #a3be8c; + --3: #b48ead; + --4: #c8ccd4; + --5: #e06c75; + --6: #61afef; + --7: #be5046; +} diff --git a/public/static/colorschemes/tokyo-night.css b/public/static/colorschemes/tokyo-night.css new file mode 100644 index 0000000..b7a30cf --- /dev/null +++ b/public/static/colorschemes/tokyo-night.css @@ -0,0 +1,11 @@ +:root { + --bg: #1a1b26; + --fg: #c0caf5; + --1: #32364a; + --2: #a9b1d6; + --3: #5a5bb8; + --4: #6b7089; + --5: #e2afff; + --6: #a9a1e1; + --7: #988bc7; +} diff --git a/public/static/themes/simple.css b/public/static/themes/simple.css index 97643e8..17962d0 100644 --- a/public/static/themes/simple.css +++ b/public/static/themes/simple.css @@ -260,3 +260,44 @@ footer { .page_navigation button:active { filter: brightness(1.2); } + +/* Styles for the about page */ + + .about-container article{ + font-size: 1.5rem; + color:var(--fg); + padding-bottom: 10px; + } + + .about-container article h1{ + color: var(--2); + font-size: 2.8rem; + } + + .about-container article div{ + padding-bottom: 15px; + } + + .about-container a{ + color:var(--3); + } + + .about-container article h2{ + color: var(--3); + font-size: 1.8rem; + padding-bottom: 10px; + } + + .about-container p{ + color:var(--fg); + font-size: 1.6rem; + padding-bottom: 10px; + } + + .about-container h3{ + font-size: 1.5rem; + } + + .about-container { + width: 80%; + } diff --git a/public/templates/about.html b/public/templates/about.html index 56c2165..9c4cbb0 100644 --- a/public/templates/about.html +++ b/public/templates/about.html @@ -1,20 +1,29 @@ {{>header this}}
-

Websurfx

- a lightening fast, privacy respecting, secure meta search engine -
- Lorem ipsum dolor sit amet, officia excepteur ex fugiat reprehenderit enim - labore culpa sint ad nisi Lorem pariatur mollit ex esse exercitation amet. - Nisi anim cupidatat excepteur officia. Reprehenderit nostrud nostrud ipsum - Lorem est aliquip amet voluptate voluptate dolor minim nulla est proident. - Nostrud officia pariatur ut officia. Sit irure elit esse ea nulla sunt ex - occaecat reprehenderit commodo officia dolor Lorem duis laboris cupidatat - officia voluptate. Culpa proident adipisicing id nulla nisi laboris ex in - Lorem sunt duis officia eiusmod. Aliqua reprehenderit commodo ex non - excepteur duis sunt velit enim. Voluptate laboris sint cupidatat ullamco ut - ea consectetur et est culpa et culpa duis. +
+
+

Websurfx

+
+
+

A modern-looking, lightning-fast, privacy-respecting, secure meta search engine written in Rust. It provides a fast and secure search experience while respecting user privacy.
It aggregates results from multiple search engines and presents them in an unbiased manner, filtering out trackers and ads. +

+ +

Some of the Top Features:

+ +
    Lightning fast - Results load within milliseconds for an instant search experience.
+ +
    Secure search - All searches are performed over an encrypted connection to prevent snooping.
+ +
    Ad free results - All search results are ad free and clutter free for a clean search experience.
+ +
    Privacy focused - Websurface does not track, store or sell your search data. Your privacy is our priority.
+ + + +
    Highly customizable - Websurface comes with 9 built-in color themes and supports creating custom themes effortlessly.
+ +

Devoloped by: Websurfx team

{{>footer}} + diff --git a/public/templates/index.html b/public/templates/index.html index 64021e9..5bb4d34 100644 --- a/public/templates/index.html +++ b/public/templates/index.html @@ -1,6 +1,6 @@ {{>header this}}
- Websurfx meta-search engine logo + Websurfx meta-search engine logo {{>search_bar}}
diff --git a/src/bin/websurfx.rs b/src/bin/websurfx.rs index fa21486..8661725 100644 --- a/src/bin/websurfx.rs +++ b/src/bin/websurfx.rs @@ -5,7 +5,6 @@ use std::net::TcpListener; -use env_logger::Env; use websurfx::{config_parser::parser::Config, run}; /// The function that launches the main server and registers all the routes of the website. @@ -20,7 +19,10 @@ async fn main() -> std::io::Result<()> { let config = Config::parse().unwrap(); // Initializing logging middleware with level set to default or info. - env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); + if config.logging || config.debug { + use env_logger::Env; + env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); + } log::info!("started server on port {}", config.port); diff --git a/src/cache/cacher.rs b/src/cache/cacher.rs index 54d9a48..87a6c6d 100644 --- a/src/cache/cacher.rs +++ b/src/cache/cacher.rs @@ -10,9 +10,8 @@ use redis::{Client, Commands, Connection}; /// # Fields /// /// * `redis_connection_url` - It stores the redis Connection url address. -#[derive(Clone)] pub struct RedisCache { - redis_connection_url: String, + connection: Connection, } impl RedisCache { @@ -21,10 +20,11 @@ impl RedisCache { /// # Arguments /// /// * `redis_connection_url` - It stores the redis Connection url address. - pub fn new(redis_connection_url: String) -> Self { - RedisCache { - redis_connection_url, - } + pub fn new(redis_connection_url: String) -> Result> { + let client = Client::open(redis_connection_url)?; + let connection = client.get_connection()?; + let redis_cache = RedisCache { connection }; + Ok(redis_cache) } /// A helper function which computes the hash of the url and formats and returns it as string. @@ -32,7 +32,7 @@ impl RedisCache { /// # Arguments /// /// * `url` - It takes an url as string. - fn compute_url_hash(self, url: &str) -> String { + fn compute_url_hash(url: &str) -> String { format!("{:?}", compute(url)) } @@ -41,11 +41,9 @@ impl RedisCache { /// # Arguments /// /// * `url` - It takes an url as a string. - pub fn cached_results_json(self, url: String) -> Result> { - let hashed_url_string = self.clone().compute_url_hash(&url); - let mut redis_connection: Connection = - Client::open(self.redis_connection_url)?.get_connection()?; - Ok(redis_connection.get(hashed_url_string)?) + pub fn cached_results_json(&mut self, url: &str) -> Result> { + let hashed_url_string = Self::compute_url_hash(url); + Ok(self.connection.get(hashed_url_string)?) } /// A function which caches the results by using the hashed `url` as the key and @@ -57,20 +55,18 @@ impl RedisCache { /// * `json_results` - It takes the json results string as an argument. /// * `url` - It takes the url as a String. pub fn cache_results( - self, + &mut self, json_results: String, - url: String, + url: &str, ) -> Result<(), Box> { - let hashed_url_string = self.clone().compute_url_hash(&url); - let mut redis_connection: Connection = - Client::open(self.redis_connection_url)?.get_connection()?; + let hashed_url_string = Self::compute_url_hash(url); // put results_json into cache - redis_connection.set(hashed_url_string.clone(), json_results)?; + self.connection.set(&hashed_url_string, json_results)?; // Set the TTL for the key to 60 seconds - redis_connection - .expire::(hashed_url_string.clone(), 60) + self.connection + .expire::(hashed_url_string, 60) .unwrap(); Ok(()) diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 91a91ca..de7dd4e 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -1 +1 @@ -pub mod cacher; +pub mod cacher; diff --git a/src/config_parser/parser.rs b/src/config_parser/parser.rs index 4625bd8..ac200cd 100644 --- a/src/config_parser/parser.rs +++ b/src/config_parser/parser.rs @@ -3,7 +3,11 @@ use super::parser_models::Style; use rlua::Lua; -use std::fs; +use std::{format, fs, path::Path}; + +// ------- Constants -------- +static COMMON_DIRECTORY_NAME: &str = "websurfx"; +static CONFIG_FILE_NAME: &str = "config.lua"; /// A named struct which stores the parsed config file options. /// @@ -20,6 +24,16 @@ pub struct Config { pub binding_ip_addr: String, pub style: Style, pub redis_connection_url: String, + pub aggregator: AggreatorConfig, + pub logging: bool, + pub debug: bool, +} + +/// Configuration options for the aggregator. +#[derive(Clone)] +pub struct AggreatorConfig { + /// Whether to introduce a random delay before sending the request to the search engine. + pub random_delay: bool, } impl Config { @@ -32,15 +46,24 @@ impl Config { /// or io error if the config.lua file doesn't exists otherwise it returns a newly contructed /// Config struct with all the parsed config options from the parsed config file. pub fn parse() -> Result> { - let lua = Lua::new(); - - lua.context(|context| { + Lua::new().context(|context| -> Result> { let globals = context.globals(); context - .load(&fs::read_to_string("./websurfx/config.lua")?) + .load(&fs::read_to_string( + Config::handle_different_config_file_path()?, + )?) .exec()?; + let production_use = globals.get::<_, bool>("production_use")?; + let aggregator_config = if production_use { + AggreatorConfig { random_delay: true } + } else { + AggreatorConfig { + random_delay: false, + } + }; + Ok(Config { port: globals.get::<_, u16>("port")?, binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?, @@ -49,7 +72,53 @@ impl Config { globals.get::<_, String>("colorscheme")?, ), redis_connection_url: globals.get::<_, String>("redis_connection_url")?, + aggregator: aggregator_config, + logging: globals.get::<_, bool>("logging")?, + debug: globals.get::<_, bool>("debug")?, }) }) } + /// A helper function which returns an appropriate config file path checking if the config + /// file exists on that path. + /// + /// # Error + /// + /// Returns a `config file not found!!` error if the config file is not present under following + /// paths which are: + /// 1. `~/.config/websurfx/` if it not present here then it fallbacks to the next one (2) + /// 2. `/etc/xdg/websurfx/config.lua` if it is not present here then it fallbacks to the next + /// one (3). + /// 3. `websurfx/` (under project folder ( or codebase in other words)) if it is not present + /// here then it returns an error as mentioned above. + fn handle_different_config_file_path() -> Result> { + if Path::new( + format!( + "{}/.config/{}/config.lua", + std::env::var("HOME").unwrap(), + COMMON_DIRECTORY_NAME + ) + .as_str(), + ) + .exists() + { + Ok(format!( + "{}/.config/{}/{}", + std::env::var("HOME").unwrap(), + COMMON_DIRECTORY_NAME, + CONFIG_FILE_NAME + )) + } else if Path::new( + format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str(), + ) + .exists() + { + Ok("/etc/xdg/websurfx/config.lua".to_string()) + } else if Path::new(format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str()) + .exists() + { + Ok("./websurfx/config.lua".to_string()) + } else { + Err(format!("Config file not found!!").into()) + } + } } diff --git a/src/handler/mod.rs b/src/handler/mod.rs new file mode 100644 index 0000000..daa5212 --- /dev/null +++ b/src/handler/mod.rs @@ -0,0 +1 @@ +pub mod public_path_handler; diff --git a/src/handler/public_path_handler.rs b/src/handler/public_path_handler.rs new file mode 100644 index 0000000..b99283e --- /dev/null +++ b/src/handler/public_path_handler.rs @@ -0,0 +1,31 @@ +//! This module provides the functionality to handle theme folder present on different paths and +//! provide one appropriate path on which it is present and can be used. + +use std::io::Error; +use std::path::Path; + +// ------- Constants -------- +static PUBLIC_DIRECTORY_NAME: &str = "public"; + +/// A function which returns an appropriate theme directory path checking if the theme +/// directory exists on that path. +/// +/// # Error +/// +/// Returns a `Theme (public) folder not found!!` error if the theme folder is not present under following +/// paths which are: +/// 1. `/opt/websurfx` if it not present here then it fallbacks to the next one (2) +/// 2. Under project folder ( or codebase in other words) if it is not present +/// here then it returns an error as mentioned above. +pub fn handle_different_public_path() -> Result { + if Path::new(format!("/opt/websurfx/{}/", PUBLIC_DIRECTORY_NAME).as_str()).exists() { + Ok(format!("/opt/websurfx/{}", PUBLIC_DIRECTORY_NAME)) + } else if Path::new(format!("./{}/", PUBLIC_DIRECTORY_NAME).as_str()).exists() { + Ok(format!("./{}", PUBLIC_DIRECTORY_NAME)) + } else { + Err(Error::new( + std::io::ErrorKind::NotFound, + "Themes (public) folder not found!!", + )) + } +} diff --git a/src/lib.rs b/src/lib.rs index c234658..6b6d4fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod cache; pub mod config_parser; pub mod engines; +pub mod handler; pub mod search_results_handler; pub mod server; @@ -15,6 +16,7 @@ use actix_files as fs; use actix_web::{dev::Server, middleware::Logger, web, App, HttpServer}; use config_parser::parser::Config; use handlebars::Handlebars; +use handler::public_path_handler::handle_different_public_path; /// Runs the web server on the provided TCP listener and returns a `Server` instance. /// @@ -39,8 +41,10 @@ use handlebars::Handlebars; pub fn run(listener: TcpListener, config: Config) -> std::io::Result { let mut handlebars: Handlebars = Handlebars::new(); + let public_folder_path: String = handle_different_public_path()?; + handlebars - .register_templates_directory(".html", "./public/templates") + .register_templates_directory(".html", format!("{}/templates", public_folder_path)) .unwrap(); let handlebars_ref: web::Data = web::Data::new(handlebars); @@ -51,8 +55,14 @@ pub fn run(listener: TcpListener, config: Config) -> std::io::Result { .app_data(web::Data::new(config.clone())) .wrap(Logger::default()) // added logging middleware for logging. // Serve images and static files (css and js files). - .service(fs::Files::new("/static", "./public/static").show_files_listing()) - .service(fs::Files::new("/images", "./public/images").show_files_listing()) + .service( + fs::Files::new("/static", format!("{}/static", public_folder_path)) + .show_files_listing(), + ) + .service( + fs::Files::new("/images", format!("{}/images", public_folder_path)) + .show_files_listing(), + ) .service(routes::robots_data) // robots.txt .service(routes::index) // index page .service(routes::search) // search page diff --git a/src/search_results_handler/aggregation_models.rs b/src/search_results_handler/aggregation_models.rs index 4fe670e..b6e6b81 100644 --- a/src/search_results_handler/aggregation_models.rs +++ b/src/search_results_handler/aggregation_models.rs @@ -116,7 +116,7 @@ impl RawSearchResult { } } -/// A named struct to store, serialize, deserialize the all the search results scraped and +/// A named struct to store, serialize, deserialize the all the search results scraped and /// aggregated from the upstream search engines. /// /// # Fields diff --git a/src/search_results_handler/aggregator.rs b/src/search_results_handler/aggregator.rs index 5133094..8b6bae3 100644 --- a/src/search_results_handler/aggregator.rs +++ b/src/search_results_handler/aggregator.rs @@ -29,6 +29,7 @@ use crate::engines::{duckduckgo, searx}; /// /// * `query` - Accepts a string to query with the above upstream search engines. /// * `page` - Accepts an u32 page number. +/// * `random_delay` - Accepts a boolean value to add a random delay before making the request. /// /// # Error /// @@ -38,14 +39,18 @@ use crate::engines::{duckduckgo, searx}; pub async fn aggregate( query: &str, page: u32, + random_delay: bool, + debug: bool, ) -> Result> { let user_agent: String = random_user_agent(); let mut result_map: HashMap = HashMap::new(); // Add a random delay before making the request. - let mut rng = rand::thread_rng(); - let delay_secs = rng.gen_range(1..10); - std::thread::sleep(Duration::from_secs(delay_secs)); + if random_delay || !debug { + let mut rng = rand::thread_rng(); + let delay_secs = rng.gen_range(1..10); + std::thread::sleep(Duration::from_secs(delay_secs)); + } // fetch results from upstream search engines simultaneously/concurrently. let (ddg_map_results, searx_map_results) = join!( diff --git a/src/search_results_handler/user_agent.rs b/src/search_results_handler/user_agent.rs index 09dd684..13166bf 100644 --- a/src/search_results_handler/user_agent.rs +++ b/src/search_results_handler/user_agent.rs @@ -1,13 +1,8 @@ //! This module provides the functionality to generate random user agent string. -use fake_useragent::{Browsers, UserAgentsBuilder}; +use fake_useragent::{Browsers, UserAgents, UserAgentsBuilder}; -/// A function to generate random user agent to improve privacy of the user. -/// -/// # Returns -/// -/// A randomly generated user agent string. -pub fn random_user_agent() -> String { +static USER_AGENTS: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| { UserAgentsBuilder::new() .cache(false) .dir("/tmp") @@ -21,6 +16,13 @@ pub fn random_user_agent() -> String { .set_mozilla(), ) .build() - .random() - .to_string() +}); + +/// A function to generate random user agent to improve privacy of the user. +/// +/// # Returns +/// +/// A randomly generated user agent string. +pub fn random_user_agent() -> String { + USER_AGENTS.random().to_string() } diff --git a/src/server/routes.rs b/src/server/routes.rs index 1ee9f35..9234d8d 100644 --- a/src/server/routes.rs +++ b/src/server/routes.rs @@ -7,6 +7,7 @@ use std::fs::read_to_string; use crate::{ cache::cacher::RedisCache, config_parser::parser::Config, + handler::public_path_handler::handle_different_public_path, search_results_handler::{aggregation_models::SearchResults, aggregator::aggregate}, }; use actix_web::{get, web, HttpRequest, HttpResponse}; @@ -73,7 +74,7 @@ pub async fn search( let params = web::Query::::from_query(req.query_string())?; //Initialize redis cache connection struct - let redis_cache = RedisCache::new(config.redis_connection_url.clone()); + let mut redis_cache = RedisCache::new(config.redis_connection_url.clone())?; match ¶ms.q { Some(query) => { if query.trim().is_empty() { @@ -81,11 +82,10 @@ pub async fn search( .insert_header(("location", "/")) .finish()) } else { - // Initialize the page url as an empty string - let mut page_url = String::new(); + let page_url: String; // Declare the page_url variable without initializing it + + // ... - // Find whether the page is valid page number if not then return - // the first page number and also construct the page_url accordingly let page = match params.page { Some(page_number) => { if page_number <= 1 { @@ -117,7 +117,7 @@ pub async fn search( }; // fetch the cached results json. - let cached_results_json = redis_cache.clone().cached_results_json(page_url.clone()); + let cached_results_json = redis_cache.cached_results_json(&page_url); // check if fetched results was indeed fetched or it was an error and if so // handle the data accordingly. match cached_results_json { @@ -128,12 +128,10 @@ pub async fn search( } Err(_) => { let mut results_json: crate::search_results_handler::aggregation_models::SearchResults = - aggregate(query, page).await?; + aggregate(query, page, config.aggregator.random_delay, config.debug).await?; results_json.add_style(config.style.clone()); - redis_cache.clone().cache_results( - serde_json::to_string(&results_json)?, - page_url.clone(), - )?; + redis_cache + .cache_results(serde_json::to_string(&results_json)?, &page_url)?; let page_content: String = hbs.render("search", &results_json)?; Ok(HttpResponse::Ok().body(page_content)) } @@ -149,7 +147,8 @@ pub async fn search( /// Handles the route of robots.txt page of the `websurfx` meta search engine website. #[get("/robots.txt")] pub async fn robots_data(_req: HttpRequest) -> Result> { - let page_content: String = read_to_string("./public/robots.txt")?; + let page_content: String = + read_to_string(format!("{}/robots.txt", handle_different_public_path()?))?; Ok(HttpResponse::Ok() .content_type("text/plain; charset=ascii") .body(page_content)) diff --git a/websurfx/config.lua b/websurfx/config.lua index 916a9b3..3daaa91 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -1,8 +1,15 @@ --- Server +-- ### General ### +logging = true -- an option to enable or disable logs. +debug = false -- an option to enable or disable debug mode. + +-- ### Server ### port = "8080" -- port on which server should be launched binding_ip_addr = "127.0.0.1" --ip address on the which server should be launched. +production_use = false -- whether to use production mode or not (in other words this option should be used if it is to be used to host it on the server to provide a service to a large number of users) +-- if production_use is set to true +-- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests. --- Website +-- ### Website ### -- The different colorschemes provided are: -- {{ -- catppuccin-mocha @@ -17,5 +24,5 @@ binding_ip_addr = "127.0.0.1" --ip address on the which server should be launche colorscheme = "catppuccin-mocha" -- the colorscheme name which should be used for the website theme theme = "simple" -- the theme name which should be used for the website --- Caching +-- ### Caching ### redis_connection_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.