92 lines
1.6 KiB
Go
92 lines
1.6 KiB
Go
package main
|
|
|
|
import (
|
|
"GofileScrapper/api"
|
|
"GofileScrapper/misc"
|
|
"GofileScrapper/structs"
|
|
"encoding/json"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type oldContent struct {
|
|
id string
|
|
retries int
|
|
}
|
|
|
|
func run(guest structs.Guest) []structs.Content {
|
|
var contents []structs.Content
|
|
|
|
old := oldContent{
|
|
id: misc.GetRandomContent(),
|
|
retries: 0,
|
|
}
|
|
|
|
for i := 0; i < misc.MaxPerThread; i++ {
|
|
if old.retries == misc.MaxRetries {
|
|
old = oldContent{
|
|
id: misc.GetRandomContent(),
|
|
retries: 0,
|
|
}
|
|
}
|
|
|
|
content, err := api.GetContent(guest, old.id)
|
|
|
|
if err != nil {
|
|
misc.Logger.Error().Msg(err.Error())
|
|
} else {
|
|
if content.Status == "error-rateLimit" {
|
|
old.retries += 1
|
|
misc.Logger.Info().Msg(fmt.Sprintf("Rate limit reached, waiting %ds...", misc.WaitingRetryTime))
|
|
time.Sleep(misc.WaitingRetryTime * time.Second)
|
|
} else {
|
|
contents = append(contents, content)
|
|
|
|
if content.Status != "error-notFound" {
|
|
out, _ := json.Marshal(content)
|
|
misc.Logger.Debug().Msg(string(out))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return contents
|
|
}
|
|
|
|
func main() {
|
|
var guests []structs.Guest
|
|
|
|
for i := 0; i < misc.GuestCount; i++ {
|
|
guest, err := api.GetGuest()
|
|
if err != nil {
|
|
misc.Logger.Error().Msg(err.Error())
|
|
} else {
|
|
guests = append(guests, guest)
|
|
}
|
|
}
|
|
|
|
results := make(chan structs.Content)
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
for _, guest := range guests {
|
|
wg.Add(1)
|
|
go func(guest structs.Guest) {
|
|
defer wg.Done()
|
|
for _, content := range run(guest) {
|
|
results <- content
|
|
}
|
|
}(guest)
|
|
}
|
|
|
|
go func() {
|
|
wg.Wait()
|
|
close(results)
|
|
}()
|
|
|
|
for content := range results {
|
|
fmt.Println("Content:", content)
|
|
}
|
|
}
|