README change

README.md

@@ -1,31 +1,54 @@
 A free to use update tool which checks phish.sinking.yachts database.
 It adds all domains found after custom domains are added. If not needed they can be commented or removed.
 
+**added a processbar**
+
+To install this package you can use one of the below.
+
+```text
+git clone https://git.codingvm.codes/ultimateplayer1999/Golang-phish-update.git (Use this one if you do not have SSH keys linked.)
+
+git clone git@git.codingvm.codes:ultimateplayer1999/Golang-phish-update.git
+```
+
+To install the needed external packages use the following commands.
+```text
+go get github.com/joho/godotenv
+go get github.com/cheggaaa/pb/v3
+go get github.com/go-sql-driver/mysql
+```
+
 The following code is needed in a .env. This is for the DB connection.
 
-> DB_HOST=
+```text
->
+DB_HOST=
-> DB_PORT=
+DB_PORT=
->
+DATABASE=
-> DATABASE=
+DB_USER=
->
+DB_PASS=
-> DB_USER=
+```
->
-> DB_PASS=
 
 This is based on the phish table. Table can also be changed if needed.
 
-Execution time is *1h23m35.23654842s*. This was for **18855 records**.
+Execution time is *1h26m19.334785368s*. This was for **18876 records**.
 
-Sample output with all defined code:
+Sample output with provided code:
-Script started at 11-01-2023 10:11:17
+
-Deletion: Number of rows affected: 2
+```text
+Script started at 12-01-2023 13:49:59
+Deletion: Number of rows affected: 59
 Altering: Number of rows affected: 0
 Insert: Number of rows affected: 1
 Insert: Number of rows affected: 1
-11-01-2023 10:11:18: Initial database table setup done
+Status code dbsize endpoint:  200
+12-01-2023 13:50:00: Initial database table setup done
 Status code:  200
-Total number of rows affected: 18855
+18876 / 18876 [-------------------------------------------------------------------------------------------->] 100%
+Total number of rows affected: 18876
 Recent changes recieved and has been added to the database. Removed domains deleted from database. Reorded domains.
-Script ended at 11-01-2023 11:34:52
+Script ended at 12-01-2023 15:16:19
-Time until completion:  1h23m35.23654842s
+Time until completion:  1h26m19.334785368s
+```
+
+# DISCLAIMER
+The binary may need to be rebuild. As it is build on Ubuntu and it may also depends on how you installed Golang.

dbupdate.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/joho/godotenv"
+	"github.com/cheggaaa/pb/v3"
 
 	_ "github.com/go-sql-driver/mysql"
 )
@@ -92,6 +93,35 @@ func main() {
 	// Define name for use later
 	// name := os.Getenv("USER_OR_BOTNAME")
 
+	// Make an HTTP request to phish.sinking.yachts dbsize endpoint
+	sizereq, err := http.NewRequest("GET", "https://phish.sinking.yachts/v2/dbsize", nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	sizereq.Header.Set("X-Identity", "ultimatebot db updater via Golang")
+	sizeclient := &http.Client{}
+	sizeresp, err := sizeclient.Do(sizereq)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer sizeresp.Body.Close()
+
+	// Request status code of endpoint
+	fmt.Println("Status code dbsize endpoint: ", sizeresp.StatusCode)
+	sizebody, err := ioutil.ReadAll(sizeresp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Unmarshal request (To be sure)
+	var sizedata int64
+	err = json.Unmarshal(sizebody, &sizedata)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	maxpbsize := sizedata
+
 	// Make an HTTP request to phish.sinking.yachts alldomains endpoint
 	req, err := http.NewRequest("GET", "https://phish.sinking.yachts/v2/all", nil)
 	if err != nil {
@@ -105,15 +135,18 @@ func main() {
 	}
 	defer resp.Body.Close()
 
+	// Setup current time variable, which will get used later
 	currentTime := time.Now()
 	formattedTime := currentTime.Format("02-01-2006 15:04:05")
 	fmt.Println(formattedTime + ": Initial database table setup done")
 
+	// Request status code of endpoint
 	fmt.Println("Status code: ", resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		log.Fatal(err)
 	}
+	// Unmarshal request
 	var data []string
 	err = json.Unmarshal(body, &data)
 	if err != nil {
@@ -123,6 +156,9 @@ func main() {
 	// Initialize a variable to keep track of the total number of rows affected
 	var totalRowsAffected int64
 
+	// Start processbar with variable from earlier with value from dbsize endpoint
+	bar := pb.Full.Start64(maxpbsize)
+
 	for _, element := range data {
 		// fmt.Println(element)
 		// Add domains
@@ -137,7 +173,13 @@ func main() {
 		}
 
 		totalRowsAffected += rowsAffected
+		bar.Increment()
+		time.Sleep(time.Millisecond)
 	}
+	// finish the processbar
+	bar.Finish()
+
+	// Print the totalRowsAffected (Which will be the same as max processbar)
 	fmt.Printf("Total number of rows affected: %d\n", totalRowsAffected)
 
 	// Get the end time
@@ -149,7 +191,7 @@ func main() {
 	fmt.Println("Recent changes recieved and has been added to the database. Removed domains deleted from database. Reorded domains.")
 
 	fmt.Println("Script ended at " + formattedEndTime)
-	
+
 	// Calculate the time difference
 	diff := EndTime.Sub(StartTime)
 
@@ -158,4 +200,4 @@ func main() {
 
 	// dummy
 	// fmt.Println("Data: ", string(body))
-}
+}

go.mod

@@ -3,6 +3,17 @@ module domain-db-updater
 go 1.19
 
 require (
-	github.com/go-sql-driver/mysql v1.7.0 // indirect
+	github.com/go-sql-driver/mysql v1.7.0 // direct
-	github.com/joho/godotenv v1.4.0 // indirect
+	github.com/joho/godotenv v1.4.0 // direct
+)
+
+require (
+	github.com/VividCortex/ewma v1.1.1 // indirect
+	github.com/cheggaaa/pb/v3 v3.1.0 // indirect
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-runewidth v0.0.12 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
 )

go.sum

@@ -1,4 +1,23 @@
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/cheggaaa/pb/v3 v3.1.0 h1:3uouEsl32RL7gTiQsuaXD4Bzbfl5tGztXGUvXbs4O04=
+github.com/cheggaaa/pb/v3 v3.1.0/go.mod h1:YjrevcBqadFDaGQKRdmZxTY42pXEqda48Ea3lt0K/BE=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
 github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=