Added a processbar

This commit is contained in:
ultimateplayer1999 2023-01-12 15:34:20 +01:00
parent deb3d75f70
commit 1781f35645
Signed by: ultimateplayer1999
GPG Key ID: 690D4E43F8B51E51
5 changed files with 87 additions and 11 deletions

View File

@ -1,6 +1,8 @@
A free to use update tool which checks phish.sinking.yachts database.
It adds all domains found after custom domains are added. If not needed they can be commented or removed.
**added a processbar**
The following code is needed in a .env. This is for the DB connection.
```text
@ -13,20 +15,22 @@ DB_PASS=
This is based on the phish table. Table can also be changed if needed.
Execution time is *1h23m35.23654842s*. This was for **18855 records**.
Execution time is *1h26m19.334785368s*. This was for **18876 records**.
Sample output with provided code:
```text
Script started at 11-01-2023 10:11:17
Deletion: Number of rows affected: 2
Script started at 12-01-2023 13:49:59
Deletion: Number of rows affected: 59
Altering: Number of rows affected: 0
Insert: Number of rows affected: 1
Insert: Number of rows affected: 1
11-01-2023 10:11:18: Initial database table setup done
Status code dbsize endpoint: 200
12-01-2023 13:50:00: Initial database table setup done
Status code: 200
Total number of rows affected: 18855
18876 / 18876 [---------------------------------------------------------------------------------------------------------------------------------------------------------------------------->] 100%
Total number of rows affected: 18876
Recent changes recieved and has been added to the database. Removed domains deleted from database. Reorded domains.
Script ended at 11-01-2023 11:34:52
Time until completion: 1h23m35.23654842s
Script ended at 12-01-2023 15:16:19
Time until completion: 1h26m19.334785368s
```

View File

@ -11,6 +11,7 @@ import (
"time"
"github.com/joho/godotenv"
"github.com/cheggaaa/pb/v3"
_ "github.com/go-sql-driver/mysql"
)
@ -92,6 +93,35 @@ func main() {
// Define name for use later
// name := os.Getenv("USER_OR_BOTNAME")
// Make an HTTP request to phish.sinking.yachts dbsize endpoint
sizereq, err := http.NewRequest("GET", "https://phish.sinking.yachts/v2/dbsize", nil)
if err != nil {
log.Fatal(err)
}
sizereq.Header.Set("X-Identity", "ultimatebot db updater via Golang")
sizeclient := &http.Client{}
sizeresp, err := sizeclient.Do(sizereq)
if err != nil {
log.Fatal(err)
}
defer sizeresp.Body.Close()
// Request status code of endpoint
fmt.Println("Status code dbsize endpoint: ", sizeresp.StatusCode)
sizebody, err := ioutil.ReadAll(sizeresp.Body)
if err != nil {
log.Fatal(err)
}
// Unmarshal request (To be sure)
var sizedata int64
err = json.Unmarshal(sizebody, &sizedata)
if err != nil {
log.Fatal(err)
}
maxpbsize := sizedata
// Make an HTTP request to phish.sinking.yachts alldomains endpoint
req, err := http.NewRequest("GET", "https://phish.sinking.yachts/v2/all", nil)
if err != nil {
@ -105,15 +135,18 @@ func main() {
}
defer resp.Body.Close()
// Setup current time variable, which will get used later
currentTime := time.Now()
formattedTime := currentTime.Format("02-01-2006 15:04:05")
fmt.Println(formattedTime + ": Initial database table setup done")
// Request status code of endpoint
fmt.Println("Status code: ", resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
// Unmarshal request
var data []string
err = json.Unmarshal(body, &data)
if err != nil {
@ -123,6 +156,9 @@ func main() {
// Initialize a variable to keep track of the total number of rows affected
var totalRowsAffected int64
// Start processbar with variable from earlier with value from dbsize endpoint
bar := pb.Full.Start64(maxpbsize)
for _, element := range data {
// fmt.Println(element)
// Add domains
@ -137,7 +173,13 @@ func main() {
}
totalRowsAffected += rowsAffected
bar.Increment()
time.Sleep(time.Millisecond)
}
// finish the processbar
bar.Finish()
// Print the totalRowsAffected (Which will be the same as max processbar)
fmt.Printf("Total number of rows affected: %d\n", totalRowsAffected)
// Get the end time

Binary file not shown.

15
go.mod
View File

@ -3,6 +3,17 @@ module domain-db-updater
go 1.19
require (
github.com/go-sql-driver/mysql v1.7.0 // indirect
github.com/joho/godotenv v1.4.0 // indirect
github.com/go-sql-driver/mysql v1.7.0 // direct
github.com/joho/godotenv v1.4.0 // direct
)
require (
github.com/VividCortex/ewma v1.1.1 // indirect
github.com/cheggaaa/pb/v3 v3.1.0 // indirect
github.com/fatih/color v1.10.0 // indirect
github.com/mattn/go-colorable v0.1.8 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/mattn/go-runewidth v0.0.12 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
)

19
go.sum
View File

@ -1,4 +1,23 @@
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/cheggaaa/pb/v3 v3.1.0 h1:3uouEsl32RL7gTiQsuaXD4Bzbfl5tGztXGUvXbs4O04=
github.com/cheggaaa/pb/v3 v3.1.0/go.mod h1:YjrevcBqadFDaGQKRdmZxTY42pXEqda48Ea3lt0K/BE=
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=