feat(core): add rate-limit handler

Makefile

@@ -5,3 +5,7 @@ all:
 .PHONY clean:
 clean:
 	rm -rfv build/*
+
+.PHONY lint:
+lint:
+	golangci-lint run .

api/content.go

@@ -11,7 +11,7 @@ import (
 func GetContent(guest structs.Guest, id string) (structs.Content, error) {
 	body, err := misc.Fetch(
 		"GET",
-		misc.API_URL+"/contents/"+id+"?wt=4fd6sg89d7s6",
+		misc.ApiUrl+"/contents/"+id+"?wt=4fd6sg89d7s6",
 		bytes.NewBuffer([]byte(`{}`)),
 		[]misc.HeaderType{
 			{Key: "Authorization", Value: fmt.Sprintf("Bearer %s", guest.Data.Token)},

api/guest.go

@@ -11,7 +11,7 @@ import (
 func GetGuest() (structs.Guest, error) {
 	body, err := misc.Fetch(
 		"POST",
-		misc.API_URL+"/accounts",
+		misc.ApiUrl+"/accounts",
 		bytes.NewBuffer([]byte(`{}`)),
 		[]misc.HeaderType{},
 	)
@@ -31,7 +31,7 @@ func GetGuest() (structs.Guest, error) {
 
 	body, err = misc.Fetch(
 		"GET",
-		misc.API_URL+"/accounts/"+preGuest.Data.Id,
+		misc.ApiUrl+"/accounts/"+preGuest.Data.Id,
 		bytes.NewBuffer([]byte(`{}`)),
 		[]misc.HeaderType{
 			{Key: "Authorization", Value: fmt.Sprintf("Bearer %s", preGuest.Data.Token)},

app.go

@@ -6,19 +6,42 @@ import (
 	"GofileScrapper/structs"
 	"fmt"
 	"sync"
+	"time"
 )
 
+type OldContent struct {
+	ID      string
+	Retried int
+}
+
 func run(guest structs.Guest) []structs.Content {
 	var contents []structs.Content
 
-	for i := 0; i < 10; i++ {
+	old := OldContent{
-		id := misc.GetRandomContent()
+		ID:      misc.GetRandomContent(),
-		content, err := api.GetContent(guest, id)
+		Retried: 0,
+	}
+
+	for i := 0; i < misc.MaxPerThread; i++ {
+		if old.Retried == misc.MaxRetries {
+			old = OldContent{
+				ID:      misc.GetRandomContent(),
+				Retried: 0,
+			}
+		}
+
+		content, err := api.GetContent(guest, old.ID)
 
 		if err != nil {
 			misc.Logger.Error().Msg(err.Error())
 		} else {
-			contents = append(contents, content)
+			if content.Status == "error-rateLimit" {
+				old.Retried += 1
+				misc.Logger.Info().Msg(fmt.Sprintf("Rate limit reached, waiting %ds...", misc.WaitingRetryTime))
+				time.Sleep(misc.WaitingRetryTime * time.Second)
+			} else {
+				contents = append(contents, content)
+			}
 		}
 	}
 
@@ -28,7 +51,7 @@ func run(guest structs.Guest) []structs.Content {
 func main() {
 	var guests []structs.Guest
 
-	for i := 0; i < misc.GUEST_COUNT; i++ {
+	for i := 0; i < misc.GuestCount; i++ {
 		guest, err := api.GetGuest()
 		if err != nil {
 			misc.Logger.Error().Msg(err.Error())

misc/constants.go

@@ -1,6 +1,11 @@
 package misc
 
-var CHARSET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+const Charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
-var API_URL = "https://api.gofile.io"
+const ApiUrl = "https://api.gofile.io"
 
-var GUEST_COUNT = 5
+const GuestCount = 5
+
+const MaxRetries = 5
+const WaitingRetryTime = 5
+
+const MaxPerThread = 10

misc/generator.go

@@ -10,11 +10,11 @@ func GetRandomContent() string {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	var result strings.Builder
-	charsetLength := len(CHARSET)
+	charsetLength := len(Charset)
 
 	for i := 0; i < 6; i++ {
 		randomIndex := r.Intn(charsetLength)
-		result.WriteByte(CHARSET[randomIndex])
+		result.WriteByte(Charset[randomIndex])
 	}
 
 	return result.String()