feat(core): add rate-limit handler
This commit is contained in:
parent
65ee6e91f5
commit
2c2871e7e0
4
Makefile
4
Makefile
|
@ -5,3 +5,7 @@ all:
|
||||||
.PHONY clean:
|
.PHONY clean:
|
||||||
clean:
|
clean:
|
||||||
rm -rfv build/*
|
rm -rfv build/*
|
||||||
|
|
||||||
|
.PHONY lint:
|
||||||
|
lint:
|
||||||
|
golangci-lint run .
|
||||||
|
|
|
@ -11,7 +11,7 @@ import (
|
||||||
func GetContent(guest structs.Guest, id string) (structs.Content, error) {
|
func GetContent(guest structs.Guest, id string) (structs.Content, error) {
|
||||||
body, err := misc.Fetch(
|
body, err := misc.Fetch(
|
||||||
"GET",
|
"GET",
|
||||||
misc.API_URL+"/contents/"+id+"?wt=4fd6sg89d7s6",
|
misc.ApiUrl+"/contents/"+id+"?wt=4fd6sg89d7s6",
|
||||||
bytes.NewBuffer([]byte(`{}`)),
|
bytes.NewBuffer([]byte(`{}`)),
|
||||||
[]misc.HeaderType{
|
[]misc.HeaderType{
|
||||||
{Key: "Authorization", Value: fmt.Sprintf("Bearer %s", guest.Data.Token)},
|
{Key: "Authorization", Value: fmt.Sprintf("Bearer %s", guest.Data.Token)},
|
||||||
|
|
|
@ -11,7 +11,7 @@ import (
|
||||||
func GetGuest() (structs.Guest, error) {
|
func GetGuest() (structs.Guest, error) {
|
||||||
body, err := misc.Fetch(
|
body, err := misc.Fetch(
|
||||||
"POST",
|
"POST",
|
||||||
misc.API_URL+"/accounts",
|
misc.ApiUrl+"/accounts",
|
||||||
bytes.NewBuffer([]byte(`{}`)),
|
bytes.NewBuffer([]byte(`{}`)),
|
||||||
[]misc.HeaderType{},
|
[]misc.HeaderType{},
|
||||||
)
|
)
|
||||||
|
@ -31,7 +31,7 @@ func GetGuest() (structs.Guest, error) {
|
||||||
|
|
||||||
body, err = misc.Fetch(
|
body, err = misc.Fetch(
|
||||||
"GET",
|
"GET",
|
||||||
misc.API_URL+"/accounts/"+preGuest.Data.Id,
|
misc.ApiUrl+"/accounts/"+preGuest.Data.Id,
|
||||||
bytes.NewBuffer([]byte(`{}`)),
|
bytes.NewBuffer([]byte(`{}`)),
|
||||||
[]misc.HeaderType{
|
[]misc.HeaderType{
|
||||||
{Key: "Authorization", Value: fmt.Sprintf("Bearer %s", preGuest.Data.Token)},
|
{Key: "Authorization", Value: fmt.Sprintf("Bearer %s", preGuest.Data.Token)},
|
||||||
|
|
31
app.go
31
app.go
|
@ -6,19 +6,42 @@ import (
|
||||||
"GofileScrapper/structs"
|
"GofileScrapper/structs"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type OldContent struct {
|
||||||
|
ID string
|
||||||
|
Retried int
|
||||||
|
}
|
||||||
|
|
||||||
func run(guest structs.Guest) []structs.Content {
|
func run(guest structs.Guest) []structs.Content {
|
||||||
var contents []structs.Content
|
var contents []structs.Content
|
||||||
|
|
||||||
|
old := OldContent{
|
||||||
|
ID: misc.GetRandomContent(),
|
||||||
|
Retried: 0,
|
||||||
|
}
|
||||||
|
|
||||||
for i := 0; i < 10; i++ {
|
for i := 0; i < 10; i++ {
|
||||||
id := misc.GetRandomContent()
|
if old.Retried == misc.MaxRetries {
|
||||||
content, err := api.GetContent(guest, id)
|
old = OldContent{
|
||||||
|
ID: misc.GetRandomContent(),
|
||||||
|
Retried: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
content, err := api.GetContent(guest, old.ID)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
misc.Logger.Error().Msg(err.Error())
|
misc.Logger.Error().Msg(err.Error())
|
||||||
} else {
|
} else {
|
||||||
contents = append(contents, content)
|
if content.Status == "error-rateLimit" {
|
||||||
|
old.Retried += 1
|
||||||
|
misc.Logger.Info().Msg(fmt.Sprintf("Rate limit reached, waiting %ds...", misc.MaxRetries))
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
} else {
|
||||||
|
contents = append(contents, content)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -28,7 +51,7 @@ func run(guest structs.Guest) []structs.Content {
|
||||||
func main() {
|
func main() {
|
||||||
var guests []structs.Guest
|
var guests []structs.Guest
|
||||||
|
|
||||||
for i := 0; i < misc.GUEST_COUNT; i++ {
|
for i := 0; i < misc.GuestCount; i++ {
|
||||||
guest, err := api.GetGuest()
|
guest, err := api.GetGuest()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
misc.Logger.Error().Msg(err.Error())
|
misc.Logger.Error().Msg(err.Error())
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
package misc
|
package misc
|
||||||
|
|
||||||
var CHARSET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
const Charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||||
var API_URL = "https://api.gofile.io"
|
const ApiUrl = "https://api.gofile.io"
|
||||||
|
|
||||||
var GUEST_COUNT = 5
|
const GuestCount = 5
|
||||||
|
|
||||||
|
const MaxRetries = 5
|
||||||
|
|
|
@ -10,11 +10,11 @@ func GetRandomContent() string {
|
||||||
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||||
|
|
||||||
var result strings.Builder
|
var result strings.Builder
|
||||||
charsetLength := len(CHARSET)
|
charsetLength := len(Charset)
|
||||||
|
|
||||||
for i := 0; i < 6; i++ {
|
for i := 0; i < 6; i++ {
|
||||||
randomIndex := r.Intn(charsetLength)
|
randomIndex := r.Intn(charsetLength)
|
||||||
result.WriteByte(CHARSET[randomIndex])
|
result.WriteByte(Charset[randomIndex])
|
||||||
}
|
}
|
||||||
|
|
||||||
return result.String()
|
return result.String()
|
||||||
|
|
Loading…
Reference in a new issue