feat(core): add rate-limit handler

This commit is contained in:
Romain J. 2024-03-20 10:10:20 +01:00
parent 65ee6e91f5
commit 0a701b19b4
6 changed files with 51 additions and 13 deletions

View file

@ -5,3 +5,7 @@ all:
.PHONY clean:
clean:
rm -rfv build/*
.PHONY lint:
lint:
golangci-lint run .

View file

@ -11,7 +11,7 @@ import (
func GetContent(guest structs.Guest, id string) (structs.Content, error) {
body, err := misc.Fetch(
"GET",
misc.API_URL+"/contents/"+id+"?wt=4fd6sg89d7s6",
misc.ApiUrl+"/contents/"+id+"?wt=4fd6sg89d7s6",
bytes.NewBuffer([]byte(`{}`)),
[]misc.HeaderType{
{Key: "Authorization", Value: fmt.Sprintf("Bearer %s", guest.Data.Token)},

View file

@ -11,7 +11,7 @@ import (
func GetGuest() (structs.Guest, error) {
body, err := misc.Fetch(
"POST",
misc.API_URL+"/accounts",
misc.ApiUrl+"/accounts",
bytes.NewBuffer([]byte(`{}`)),
[]misc.HeaderType{},
)
@ -31,7 +31,7 @@ func GetGuest() (structs.Guest, error) {
body, err = misc.Fetch(
"GET",
misc.API_URL+"/accounts/"+preGuest.Data.Id,
misc.ApiUrl+"/accounts/"+preGuest.Data.Id,
bytes.NewBuffer([]byte(`{}`)),
[]misc.HeaderType{
{Key: "Authorization", Value: fmt.Sprintf("Bearer %s", preGuest.Data.Token)},

39
app.go
View file

@ -4,21 +4,50 @@ import (
"GofileScrapper/api"
"GofileScrapper/misc"
"GofileScrapper/structs"
"encoding/json"
"fmt"
"sync"
"time"
)
type oldContent struct {
id string
retries int
}
func run(guest structs.Guest) []structs.Content {
var contents []structs.Content
for i := 0; i < 10; i++ {
id := misc.GetRandomContent()
content, err := api.GetContent(guest, id)
old := oldContent{
id: misc.GetRandomContent(),
retries: 0,
}
for i := 0; i < misc.MaxPerThread; i++ {
if old.retries == misc.MaxRetries {
old = oldContent{
id: misc.GetRandomContent(),
retries: 0,
}
}
content, err := api.GetContent(guest, old.id)
if err != nil {
misc.Logger.Error().Msg(err.Error())
} else {
contents = append(contents, content)
if content.Status == "error-rateLimit" {
old.retries += 1
misc.Logger.Info().Msg(fmt.Sprintf("Rate limit reached, waiting %ds...", misc.WaitingRetryTime))
time.Sleep(misc.WaitingRetryTime * time.Second)
} else {
contents = append(contents, content)
if content.Status != "error-notFound" {
out, _ := json.Marshal(content)
misc.Logger.Debug().Msg(string(out))
}
}
}
}
@ -28,7 +57,7 @@ func run(guest structs.Guest) []structs.Content {
func main() {
var guests []structs.Guest
for i := 0; i < misc.GUEST_COUNT; i++ {
for i := 0; i < misc.GuestCount; i++ {
guest, err := api.GetGuest()
if err != nil {
misc.Logger.Error().Msg(err.Error())

View file

@ -1,6 +1,11 @@
package misc
var CHARSET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
var API_URL = "https://api.gofile.io"
const Charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
const ApiUrl = "https://api.gofile.io"
var GUEST_COUNT = 5
const GuestCount = 5
const MaxRetries = 5
const WaitingRetryTime = 5
const MaxPerThread = 30

View file

@ -10,11 +10,11 @@ func GetRandomContent() string {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
var result strings.Builder
charsetLength := len(CHARSET)
charsetLength := len(Charset)
for i := 0; i < 6; i++ {
randomIndex := r.Intn(charsetLength)
result.WriteByte(CHARSET[randomIndex])
result.WriteByte(Charset[randomIndex])
}
return result.String()