diff --git a/Makefile b/Makefile index 1fca6c2..9e0194a 100644 --- a/Makefile +++ b/Makefile @@ -5,3 +5,7 @@ all: .PHONY clean: clean: rm -rfv build/* + +.PHONY lint: +lint: + golangci-lint run . diff --git a/api/content.go b/api/content.go index d5862bd..a323af5 100644 --- a/api/content.go +++ b/api/content.go @@ -11,7 +11,7 @@ import ( func GetContent(guest structs.Guest, id string) (structs.Content, error) { body, err := misc.Fetch( "GET", - misc.API_URL+"/contents/"+id+"?wt=4fd6sg89d7s6", + misc.ApiUrl+"/contents/"+id+"?wt=4fd6sg89d7s6", bytes.NewBuffer([]byte(`{}`)), []misc.HeaderType{ {Key: "Authorization", Value: fmt.Sprintf("Bearer %s", guest.Data.Token)}, diff --git a/api/guest.go b/api/guest.go index eb4d739..199df17 100644 --- a/api/guest.go +++ b/api/guest.go @@ -11,7 +11,7 @@ import ( func GetGuest() (structs.Guest, error) { body, err := misc.Fetch( "POST", - misc.API_URL+"/accounts", + misc.ApiUrl+"/accounts", bytes.NewBuffer([]byte(`{}`)), []misc.HeaderType{}, ) @@ -31,7 +31,7 @@ func GetGuest() (structs.Guest, error) { body, err = misc.Fetch( "GET", - misc.API_URL+"/accounts/"+preGuest.Data.Id, + misc.ApiUrl+"/accounts/"+preGuest.Data.Id, bytes.NewBuffer([]byte(`{}`)), []misc.HeaderType{ {Key: "Authorization", Value: fmt.Sprintf("Bearer %s", preGuest.Data.Token)}, diff --git a/app.go b/app.go index d18a0de..0b51ac0 100644 --- a/app.go +++ b/app.go @@ -4,21 +4,50 @@ import ( "GofileScrapper/api" "GofileScrapper/misc" "GofileScrapper/structs" + "encoding/json" "fmt" "sync" + "time" ) +type oldContent struct { + id string + retries int +} + func run(guest structs.Guest) []structs.Content { var contents []structs.Content - for i := 0; i < 10; i++ { - id := misc.GetRandomContent() - content, err := api.GetContent(guest, id) + old := oldContent{ + id: misc.GetRandomContent(), + retries: 0, + } + + for i := 0; i < misc.MaxPerThread; i++ { + if old.retries == misc.MaxRetries { + old = oldContent{ + id: misc.GetRandomContent(), + retries: 0, + } + } + + content, err := api.GetContent(guest, old.id) if err != nil { misc.Logger.Error().Msg(err.Error()) } else { - contents = append(contents, content) + if content.Status == "error-rateLimit" { + old.retries += 1 + misc.Logger.Info().Msg(fmt.Sprintf("Rate limit reached, waiting %ds...", misc.WaitingRetryTime)) + time.Sleep(misc.WaitingRetryTime * time.Second) + } else { + contents = append(contents, content) + + if content.Status != "error-notFound" { + out, _ := json.Marshal(content) + misc.Logger.Debug().Msg(string(out)) + } + } } } @@ -28,7 +57,7 @@ func run(guest structs.Guest) []structs.Content { func main() { var guests []structs.Guest - for i := 0; i < misc.GUEST_COUNT; i++ { + for i := 0; i < misc.GuestCount; i++ { guest, err := api.GetGuest() if err != nil { misc.Logger.Error().Msg(err.Error()) diff --git a/misc/constants.go b/misc/constants.go index 5eb711f..e239994 100644 --- a/misc/constants.go +++ b/misc/constants.go @@ -1,6 +1,11 @@ package misc -var CHARSET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" -var API_URL = "https://api.gofile.io" +const Charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" +const ApiUrl = "https://api.gofile.io" -var GUEST_COUNT = 5 +const GuestCount = 5 + +const MaxRetries = 5 +const WaitingRetryTime = 5 + +const MaxPerThread = 30 diff --git a/misc/generator.go b/misc/generator.go index 64c3ae7..cb12067 100644 --- a/misc/generator.go +++ b/misc/generator.go @@ -10,11 +10,11 @@ func GetRandomContent() string { r := rand.New(rand.NewSource(time.Now().UnixNano())) var result strings.Builder - charsetLength := len(CHARSET) + charsetLength := len(Charset) for i := 0; i < 6; i++ { randomIndex := r.Intn(charsetLength) - result.WriteByte(CHARSET[randomIndex]) + result.WriteByte(Charset[randomIndex]) } return result.String()