diff --git a/README.md b/README.md index 1910d10..964c9ba 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ A free to use update tool which checks phish.sinking.yachts database. It adds all domains found after custom domains are added. If not needed they can be commented or removed. +**added a processbar** + The following code is needed in a .env. This is for the DB connection. ```text @@ -13,20 +15,22 @@ DB_PASS= This is based on the phish table. Table can also be changed if needed. -Execution time is *1h23m35.23654842s*. This was for **18855 records**. +Execution time is *1h26m19.334785368s*. This was for **18876 records**. Sample output with provided code: ```text -Script started at 11-01-2023 10:11:17 -Deletion: Number of rows affected: 2 +Script started at 12-01-2023 13:49:59 +Deletion: Number of rows affected: 59 Altering: Number of rows affected: 0 Insert: Number of rows affected: 1 Insert: Number of rows affected: 1 -11-01-2023 10:11:18: Initial database table setup done +Status code dbsize endpoint: 200 +12-01-2023 13:50:00: Initial database table setup done Status code: 200 -Total number of rows affected: 18855 +18876 / 18876 [---------------------------------------------------------------------------------------------------------------------------------------------------------------------------->] 100% +Total number of rows affected: 18876 Recent changes recieved and has been added to the database. Removed domains deleted from database. Reorded domains. -Script ended at 11-01-2023 11:34:52 -Time until completion: 1h23m35.23654842s +Script ended at 12-01-2023 15:16:19 +Time until completion: 1h26m19.334785368s ``` diff --git a/dbupdate.go b/dbupdate.go index 8909a5a..02160ce 100644 --- a/dbupdate.go +++ b/dbupdate.go @@ -11,6 +11,7 @@ import ( "time" "github.com/joho/godotenv" + "github.com/cheggaaa/pb/v3" _ "github.com/go-sql-driver/mysql" ) @@ -92,6 +93,35 @@ func main() { // Define name for use later // name := os.Getenv("USER_OR_BOTNAME") + // Make an HTTP request to phish.sinking.yachts dbsize endpoint + sizereq, err := http.NewRequest("GET", "https://phish.sinking.yachts/v2/dbsize", nil) + if err != nil { + log.Fatal(err) + } + sizereq.Header.Set("X-Identity", "ultimatebot db updater via Golang") + sizeclient := &http.Client{} + sizeresp, err := sizeclient.Do(sizereq) + if err != nil { + log.Fatal(err) + } + defer sizeresp.Body.Close() + + // Request status code of endpoint + fmt.Println("Status code dbsize endpoint: ", sizeresp.StatusCode) + sizebody, err := ioutil.ReadAll(sizeresp.Body) + if err != nil { + log.Fatal(err) + } + + // Unmarshal request (To be sure) + var sizedata int64 + err = json.Unmarshal(sizebody, &sizedata) + if err != nil { + log.Fatal(err) + } + + maxpbsize := sizedata + // Make an HTTP request to phish.sinking.yachts alldomains endpoint req, err := http.NewRequest("GET", "https://phish.sinking.yachts/v2/all", nil) if err != nil { @@ -105,15 +135,18 @@ func main() { } defer resp.Body.Close() + // Setup current time variable, which will get used later currentTime := time.Now() formattedTime := currentTime.Format("02-01-2006 15:04:05") fmt.Println(formattedTime + ": Initial database table setup done") + // Request status code of endpoint fmt.Println("Status code: ", resp.StatusCode) body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Fatal(err) } + // Unmarshal request var data []string err = json.Unmarshal(body, &data) if err != nil { @@ -123,6 +156,9 @@ func main() { // Initialize a variable to keep track of the total number of rows affected var totalRowsAffected int64 + // Start processbar with variable from earlier with value from dbsize endpoint + bar := pb.Full.Start64(maxpbsize) + for _, element := range data { // fmt.Println(element) // Add domains @@ -137,7 +173,13 @@ func main() { } totalRowsAffected += rowsAffected + bar.Increment() + time.Sleep(time.Millisecond) } + // finish the processbar + bar.Finish() + + // Print the totalRowsAffected (Which will be the same as max processbar) fmt.Printf("Total number of rows affected: %d\n", totalRowsAffected) // Get the end time @@ -149,7 +191,7 @@ func main() { fmt.Println("Recent changes recieved and has been added to the database. Removed domains deleted from database. Reorded domains.") fmt.Println("Script ended at " + formattedEndTime) - + // Calculate the time difference diff := EndTime.Sub(StartTime) @@ -158,4 +200,4 @@ func main() { // dummy // fmt.Println("Data: ", string(body)) -} \ No newline at end of file +} diff --git a/domain-db-updater b/domain-db-updater index 5dc8093..3b50c9d 100755 Binary files a/domain-db-updater and b/domain-db-updater differ diff --git a/go.mod b/go.mod index a13b800..aa3ef39 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,17 @@ module domain-db-updater go 1.19 require ( - github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/joho/godotenv v1.4.0 // indirect + github.com/go-sql-driver/mysql v1.7.0 // direct + github.com/joho/godotenv v1.4.0 // direct +) + +require ( + github.com/VividCortex/ewma v1.1.1 // indirect + github.com/cheggaaa/pb/v3 v3.1.0 // indirect + github.com/fatih/color v1.10.0 // indirect + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect + github.com/mattn/go-runewidth v0.0.12 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect ) diff --git a/go.sum b/go.sum index 7ffa482..fe455c6 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,23 @@ +github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/cheggaaa/pb/v3 v3.1.0 h1:3uouEsl32RL7gTiQsuaXD4Bzbfl5tGztXGUvXbs4O04= +github.com/cheggaaa/pb/v3 v3.1.0/go.mod h1:YjrevcBqadFDaGQKRdmZxTY42pXEqda48Ea3lt0K/BE= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=