README change

Signed-off-by: ultimateplayer1999 <ultimateplayer1999@gmail.com>

LICENSE

@@ -672,3 +672,4 @@ may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
 <https://www.gnu.org/licenses/why-not-lgpl.html>.
+

README.md

@@ -1,16 +1,54 @@
 A free to use update tool which checks phish.sinking.yachts database.
 It adds all domains found after custom domains are added. If not needed they can be commented or removed.
 
+**added a processbar**
+
+To install this package you can use one of the below.
+
+```text
+git clone https://git.codingvm.codes/ultimateplayer1999/Golang-phish-update.git (Use this one if you do not have SSH keys linked.)
+
+git clone git@git.codingvm.codes:ultimateplayer1999/Golang-phish-update.git
+```
+
+To install the needed external packages use the following commands.
+```text
+go get github.com/joho/godotenv
+go get github.com/cheggaaa/pb/v3
+go get github.com/go-sql-driver/mysql
+```
+
 The following code is needed in a .env. This is for the DB connection.
 
-> DB_HOST=
->
-> DB_PORT=
->
-> DATABASE=
->
-> DB_USER=
->
-> DB_PASS=
+```text
+DB_HOST=
+DB_PORT=
+DATABASE=
+DB_USER=
+DB_PASS=
+```
 
 This is based on the phish table. Table can also be changed if needed.
+
+Execution time is *1h26m19.334785368s*. This was for **18876 records**.
+
+Sample output with provided code:
+
+```text
+Script started at 12-01-2023 13:49:59
+Deletion: Number of rows affected: 59
+Altering: Number of rows affected: 0
+Insert: Number of rows affected: 1
+Insert: Number of rows affected: 1
+Status code dbsize endpoint:  200
+12-01-2023 13:50:00: Initial database table setup done
+Status code:  200
+18876 / 18876 [-------------------------------------------------------------------------------------------->] 100%
+Total number of rows affected: 18876
+Recent changes recieved and has been added to the database. Removed domains deleted from database. Reorded domains.
+Script ended at 12-01-2023 15:16:19
+Time until completion:  1h26m19.334785368s
+```
+
+# DISCLAIMER
+The binary may need to be rebuild. As it is build on Ubuntu and it may also depends on how you installed Golang.

dbupdate.go

@@ -8,13 +8,17 @@ import (
 	"net/http"
 	"io/ioutil"
 	"encoding/json"
+	"time"
 
 	"github.com/joho/godotenv"
+	"github.com/cheggaaa/pb/v3"
 
 	_ "github.com/go-sql-driver/mysql"
 )
 
 func main() {
+	// Clear the screen
+	fmt.Print("\033[2J")
 	// Load environment variables from .env file
 	err := godotenv.Load()
 	if err != nil {
@@ -33,6 +37,13 @@ func main() {
 		log.Fatal(err)
 	}
 
+	// Get the start time
+	StartTime := time.Now()
+
+	// Format the start time
+	formattedStartTime := StartTime.Format("02-01-2006 15:04:05")
+
+	fmt.Println("Script started at " + formattedStartTime)
 	// Perform a cleanup on DB
 	deleter, err := db.Exec("DELETE FROM phish;")
 	if err != nil {
@@ -43,7 +54,7 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	fmt.Printf("Number of rows affected: %d\n", rowsAffected)
+	fmt.Printf("Deletion: Number of rows affected: %d\n", rowsAffected)
 
 	// Perform a cleanup on DB
 	aiupdate, err := db.Exec("ALTER TABLE phish AUTO_INCREMENT = 0;")
@@ -55,7 +66,7 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	fmt.Printf("Number of rows affected: %d\n", rowsAffected)
+	fmt.Printf("Altering: Number of rows affected: %d\n", rowsAffected)
 
 	// Add local records first
 	insert1, err := db.Exec("INSERT INTO phish (Domain, Reporter, Evidence) VALUES ('iriseperiplo.info', 'Henk', 'https://www.virustotal.com/gui/url/318ac29af0001f6678efdd94c16d3225d78369123ebbbe5f50387b208e0ac523?nocache=1');")
@@ -71,17 +82,46 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	fmt.Printf("Number of rows affected: %d\n", rowsAffected)
+	fmt.Printf("Insert: Number of rows affected: %d\n", rowsAffected)
 	// Get the number of affected rows
 	rowsAffected, err = insert2.RowsAffected()
 	if err != nil {
 		log.Fatal(err)
 	}
-	fmt.Printf("Number of rows affected: %d\n", rowsAffected)
+	fmt.Printf("Insert: Number of rows affected: %d\n", rowsAffected)
 
 	// Define name for use later
 	// name := os.Getenv("USER_OR_BOTNAME")
 
+	// Make an HTTP request to phish.sinking.yachts dbsize endpoint
+	sizereq, err := http.NewRequest("GET", "https://phish.sinking.yachts/v2/dbsize", nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	sizereq.Header.Set("X-Identity", "ultimatebot db updater via Golang")
+	sizeclient := &http.Client{}
+	sizeresp, err := sizeclient.Do(sizereq)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer sizeresp.Body.Close()
+
+	// Request status code of endpoint
+	fmt.Println("Status code dbsize endpoint: ", sizeresp.StatusCode)
+	sizebody, err := ioutil.ReadAll(sizeresp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Unmarshal request (To be sure)
+	var sizedata int64
+	err = json.Unmarshal(sizebody, &sizedata)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	maxpbsize := sizedata
+
 	// Make an HTTP request to phish.sinking.yachts alldomains endpoint
 	req, err := http.NewRequest("GET", "https://phish.sinking.yachts/v2/all", nil)
 	if err != nil {
@@ -95,11 +135,18 @@ func main() {
 	}
 	defer resp.Body.Close()
 
+	// Setup current time variable, which will get used later
+	currentTime := time.Now()
+	formattedTime := currentTime.Format("02-01-2006 15:04:05")
+	fmt.Println(formattedTime + ": Initial database table setup done")
+
+	// Request status code of endpoint
 	fmt.Println("Status code: ", resp.StatusCode)
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		log.Fatal(err)
 	}
+	// Unmarshal request
 	var data []string
 	err = json.Unmarshal(body, &data)
 	if err != nil {
@@ -109,6 +156,9 @@ func main() {
 	// Initialize a variable to keep track of the total number of rows affected
 	var totalRowsAffected int64
 
+	// Start processbar with variable from earlier with value from dbsize endpoint
+	bar := pb.Full.Start64(maxpbsize)
+
 	for _, element := range data {
 		// fmt.Println(element)
 		// Add domains
@@ -123,8 +173,31 @@ func main() {
 		}
 
 		totalRowsAffected += rowsAffected
+		bar.Increment()
+		time.Sleep(time.Millisecond)
 	}
+	// finish the processbar
+	bar.Finish()
+
+	// Print the totalRowsAffected (Which will be the same as max processbar)
 	fmt.Printf("Total number of rows affected: %d\n", totalRowsAffected)
+
+	// Get the end time
+	EndTime := time.Now()
+
+	// Format the end time
+	formattedEndTime := EndTime.Format("02-01-2006 15:04:05")
+
+	fmt.Println("Recent changes recieved and has been added to the database. Removed domains deleted from database. Reorded domains.")
+
+	fmt.Println("Script ended at " + formattedEndTime)
+
+	// Calculate the time difference
+	diff := EndTime.Sub(StartTime)
+
+	// Print the time difference
+	fmt.Println("Time until completion: ", diff)
+
 	// dummy
 	// fmt.Println("Data: ", string(body))
 }

go.mod

@@ -1,8 +1,19 @@
-module skywolf.nl/domain-db-updater
+module domain-db-updater
 
 go 1.19
 
 require (
-	github.com/go-sql-driver/mysql v1.7.0 // indirect
-	github.com/joho/godotenv v1.4.0 // indirect
+	github.com/go-sql-driver/mysql v1.7.0 // direct
+	github.com/joho/godotenv v1.4.0 // direct
+)
+
+require (
+	github.com/VividCortex/ewma v1.1.1 // indirect
+	github.com/cheggaaa/pb/v3 v3.1.0 // indirect
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-runewidth v0.0.12 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
 )

go.sum

@@ -1,4 +1,23 @@
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/cheggaaa/pb/v3 v3.1.0 h1:3uouEsl32RL7gTiQsuaXD4Bzbfl5tGztXGUvXbs4O04=
+github.com/cheggaaa/pb/v3 v3.1.0/go.mod h1:YjrevcBqadFDaGQKRdmZxTY42pXEqda48Ea3lt0K/BE=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
 github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
 github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=