package main import ( "GofileScrapper/api" "GofileScrapper/misc" "GofileScrapper/structs" "encoding/json" "fmt" "sync" "time" ) type oldContent struct { id string retries int } func run(guest structs.Guest) []structs.Content { var contents []structs.Content old := oldContent{ id: misc.GetRandomContent(), retries: 0, } for i := 0; i < misc.MaxPerThread; i++ { if old.retries == misc.MaxRetries { old = oldContent{ id: misc.GetRandomContent(), retries: 0, } } content, err := api.GetContent(guest, old.id) if err != nil { misc.Logger.Error().Msg(err.Error()) } else { if content.Status == "error-rateLimit" { old.retries += 1 misc.Logger.Info().Msg(fmt.Sprintf("Rate limit reached, waiting %ds...", misc.WaitingRetryTime)) time.Sleep(misc.WaitingRetryTime * time.Second) } else { contents = append(contents, content) if content.Status != "error-notFound" { out, _ := json.Marshal(content) misc.Logger.Debug().Msg(string(out)) } } } } return contents } func main() { var guests []structs.Guest for i := 0; i < misc.GuestCount; i++ { guest, err := api.GetGuest() if err != nil { misc.Logger.Error().Msg(err.Error()) } else { guests = append(guests, guest) } } results := make(chan structs.Content) var wg sync.WaitGroup for _, guest := range guests { wg.Add(1) go func(guest structs.Guest) { defer wg.Done() for _, content := range run(guest) { results <- content } }(guest) } go func() { wg.Wait() close(results) }() for content := range results { fmt.Println("Content:", content) } }