ultimateplayer1999/Golang-phish-update
1
1
Fork 0
mirror of https://codeberg.org/ultimateplayer1999/Golang-phish-update.git synced 2024-11-22 05:08:21 -05:00
Code Issues Releases Activity

Added a processbar

Browse Source
This commit is contained in:
ultimateplayer1999 2023-01-12 15:34:20 +01:00
parent deb3d75f70
commit 1781f35645
Signed by: ultimateplayer1999
GPG Key ID: 690D4E43F8B51E51
5 changed files with 87 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
README.md
View File

@ -1,6 +1,8 @@
A free to use update tool which checks phish.sinking.yachts database. A free to use update tool which checks phish.sinking.yachts database.
It adds all domains found after custom domains are added. If not needed they can be commented or removed. It adds all domains found after custom domains are added. If not needed they can be commented or removed.
**added a processbar**
The following code is needed in a .env. This is for the DB connection. The following code is needed in a .env. This is for the DB connection.
```text ```text
@ -13,20 +15,22 @@ DB_PASS=
This is based on the phish table. Table can also be changed if needed. This is based on the phish table. Table can also be changed if needed.
Execution time is *1h23m35.23654842s*. This was for **18855 records**. Execution time is *1h26m19.334785368s*. This was for **18876 records**.
Sample output with provided code: Sample output with provided code:
```text ```text
Script started at 11-01-2023 10:11:17 Script started at 12-01-2023 13:49:59
Deletion: Number of rows affected: 2 Deletion: Number of rows affected: 59
Altering: Number of rows affected: 0 Altering: Number of rows affected: 0
Insert: Number of rows affected: 1 Insert: Number of rows affected: 1
Insert: Number of rows affected: 1 Insert: Number of rows affected: 1
11-01-2023 10:11:18: Initial database table setup done Status code dbsize endpoint: 200
12-01-2023 13:50:00: Initial database table setup done
Status code: 200 Status code: 200
Total number of rows affected: 18855 18876 / 18876 [---------------------------------------------------------------------------------------------------------------------------------------------------------------------------->] 100%
Total number of rows affected: 18876
Recent changes recieved and has been added to the database. Removed domains deleted from database. Reorded domains. Recent changes recieved and has been added to the database. Removed domains deleted from database. Reorded domains.
Script ended at 11-01-2023 11:34:52 Script ended at 12-01-2023 15:16:19
Time until completion: 1h23m35.23654842s Time until completion: 1h26m19.334785368s
``` ```

42
dbupdate.go
View File

@ -11,6 +11,7 @@ import (
"time" "time"
"github.com/joho/godotenv" "github.com/joho/godotenv"
"github.com/cheggaaa/pb/v3"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
) )
@ -92,6 +93,35 @@ func main() {
// Define name for use later // Define name for use later
// name := os.Getenv("USER_OR_BOTNAME") // name := os.Getenv("USER_OR_BOTNAME")
// Make an HTTP request to phish.sinking.yachts dbsize endpoint
sizereq, err := http.NewRequest("GET", "https://phish.sinking.yachts/v2/dbsize", nil)
if err != nil {
log.Fatal(err)
}
sizereq.Header.Set("X-Identity", "ultimatebot db updater via Golang")
sizeclient := &http.Client{}
sizeresp, err := sizeclient.Do(sizereq)
if err != nil {
log.Fatal(err)
}
defer sizeresp.Body.Close()
// Request status code of endpoint
fmt.Println("Status code dbsize endpoint: ", sizeresp.StatusCode)
sizebody, err := ioutil.ReadAll(sizeresp.Body)
if err != nil {
log.Fatal(err)
}
// Unmarshal request (To be sure)
var sizedata int64
err = json.Unmarshal(sizebody, &sizedata)
if err != nil {
log.Fatal(err)
}
maxpbsize := sizedata
// Make an HTTP request to phish.sinking.yachts alldomains endpoint // Make an HTTP request to phish.sinking.yachts alldomains endpoint
req, err := http.NewRequest("GET", "https://phish.sinking.yachts/v2/all", nil) req, err := http.NewRequest("GET", "https://phish.sinking.yachts/v2/all", nil)
if err != nil { if err != nil {
@ -105,15 +135,18 @@ func main() {
} }
defer resp.Body.Close() defer resp.Body.Close()
// Setup current time variable, which will get used later
currentTime := time.Now() currentTime := time.Now()
formattedTime := currentTime.Format("02-01-2006 15:04:05") formattedTime := currentTime.Format("02-01-2006 15:04:05")
fmt.Println(formattedTime + ": Initial database table setup done") fmt.Println(formattedTime + ": Initial database table setup done")
// Request status code of endpoint
fmt.Println("Status code: ", resp.StatusCode) fmt.Println("Status code: ", resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
// Unmarshal request
var data []string var data []string
err = json.Unmarshal(body, &data) err = json.Unmarshal(body, &data)
if err != nil { if err != nil {
@ -123,6 +156,9 @@ func main() {
// Initialize a variable to keep track of the total number of rows affected // Initialize a variable to keep track of the total number of rows affected
var totalRowsAffected int64 var totalRowsAffected int64
// Start processbar with variable from earlier with value from dbsize endpoint
bar := pb.Full.Start64(maxpbsize)
for _, element := range data { for _, element := range data {
// fmt.Println(element) // fmt.Println(element)
// Add domains // Add domains
@ -137,7 +173,13 @@ func main() {
} }
totalRowsAffected += rowsAffected totalRowsAffected += rowsAffected
bar.Increment()
time.Sleep(time.Millisecond)
} }
// finish the processbar
bar.Finish()
// Print the totalRowsAffected (Which will be the same as max processbar)
fmt.Printf("Total number of rows affected: %d\n", totalRowsAffected) fmt.Printf("Total number of rows affected: %d\n", totalRowsAffected)
// Get the end time // Get the end time

BIN
domain-db-updater
View File

Binary file not shown.

15
go.mod
View File

@ -3,6 +3,17 @@ module domain-db-updater
go 1.19 go 1.19
require ( require (
github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // direct
github.com/joho/godotenv v1.4.0 // indirect github.com/joho/godotenv v1.4.0 // direct
)
require (
github.com/VividCortex/ewma v1.1.1 // indirect
github.com/cheggaaa/pb/v3 v3.1.0 // indirect
github.com/fatih/color v1.10.0 // indirect
github.com/mattn/go-colorable v0.1.8 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/mattn/go-runewidth v0.0.12 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
) )

19
go.sum
View File

@ -1,4 +1,23 @@
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/cheggaaa/pb/v3 v3.1.0 h1:3uouEsl32RL7gTiQsuaXD4Bzbfl5tGztXGUvXbs4O04=
github.com/cheggaaa/pb/v3 v3.1.0/go.mod h1:YjrevcBqadFDaGQKRdmZxTY42pXEqda48Ea3lt0K/BE=
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=