new: feat: add parallelism

api/comando_torrents.go

@@ -200,47 +200,57 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
 		}
 	})
 
+	var chanIndexedTorrent = make(chan IndexedTorrent)
+
 	// for each magnet link, create a new indexed torrent
 	for _, magnetLink := range magnetLinks {
-		releaseTitle := extractReleaseName(magnetLink)
+		go func(magnetLink string) {
-		magnetAudio := []schema.Audio{}
+			releaseTitle := extractReleaseName(magnetLink)
-		if strings.Contains(strings.ToLower(releaseTitle), "dual") {
+			magnetAudio := []schema.Audio{}
-			magnetAudio = append(magnetAudio, audio...)
+			if strings.Contains(strings.ToLower(releaseTitle), "dual") {
-		} else if len(audio) > 1 {
+				magnetAudio = append(magnetAudio, audio...)
-			// remove portuguese audio, and append to magnetAudio
+			} else if len(audio) > 1 {
-			for _, a := range audio {
+				// remove portuguese audio, and append to magnetAudio
-				if a != schema.AudioPortuguese {
+				for _, a := range audio {
-					magnetAudio = append(magnetAudio, a)
+					if a != schema.AudioPortuguese {
+						magnetAudio = append(magnetAudio, a)
+					}
 				}
+			} else {
+				magnetAudio = append(magnetAudio, audio...)
 			}
-		} else {
+			// decode url encoded title
-			magnetAudio = append(magnetAudio, audio...)
+			releaseTitle, _ = url.QueryUnescape(releaseTitle)
-		}
-		// decode url encoded title
-		releaseTitle, _ = url.QueryUnescape(releaseTitle)
 
-		infoHash := extractInfoHash(magnetLink)
+			infoHash := extractInfoHash(magnetLink)
-		trackers := extractTrackers(magnetLink)
+			trackers := extractTrackers(magnetLink)
-		peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, infoHash, trackers)
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, infoHash, trackers)
-		if err != nil {
+			if err != nil {
-			fmt.Println(err)
+				fmt.Println(err)
-		}
+			}
 
-		title := processTitle(title, magnetAudio)
+			title := processTitle(title, magnetAudio)
 
-		indexedTorrents = append(indexedTorrents, IndexedTorrent{
+			it := IndexedTorrent{
-			Title:         releaseTitle,
+				Title:         releaseTitle,
-			OriginalTitle: title,
+				OriginalTitle: title,
-			Details:       link,
+				Details:       link,
-			Year:          year,
+				Year:          year,
-			Audio:         magnetAudio,
+				Audio:         magnetAudio,
-			MagnetLink:    magnetLink,
+				MagnetLink:    magnetLink,
-			Date:          date,
+				Date:          date,
-			InfoHash:      infoHash,
+				InfoHash:      infoHash,
-			Trackers:      trackers,
+				Trackers:      trackers,
-			LeechCount:    peer,
+				LeechCount:    peer,
-			SeedCount:     seed,
+				SeedCount:     seed,
-		})
+			}
+			chanIndexedTorrent <- it
+		}(magnetLink)
+	}
+
+	for i := 0; i < len(magnetLinks); i++ {
+		it := <-chanIndexedTorrent
+		indexedTorrents = append(indexedTorrents, it)
 	}
 
 	return indexedTorrents, nil

scrape/info.go

@@ -9,24 +9,29 @@ import (
 	"github.com/felipemarinho97/torrent-indexer/cache"
 )
 
+type peers struct {
+	Seeders  int `json:"seed"`
+	Leechers int `json:"leech"`
+}
+
 func getPeersFromCache(ctx context.Context, r *cache.Redis, infoHash string) (int, int, error) {
 	// get peers and seeds from redis first
 	peersCache, err := r.Get(ctx, infoHash)
 	if err == nil {
-		var peers map[string]int
+		var peers peers
 		err = json.Unmarshal(peersCache, &peers)
 		if err != nil {
 			return 0, 0, err
 		}
-		return peers["leech"], peers["seed"], nil
+		return peers.Leechers, peers.Seeders, nil
 	}
 	return 0, 0, err
 }
 
 func setPeersToCache(ctx context.Context, r *cache.Redis, infoHash string, peer, seed int) error {
-	peers := map[string]int{
+	peers := peers{
-		"leech": peer,
+		Seeders:  seed,
-		"seed":  seed,
+		Leechers: peer,
 	}
 	peersJSON, err := json.Marshal(peers)
 	if err != nil {
@@ -40,7 +45,6 @@ func setPeersToCache(ctx context.Context, r *cache.Redis, infoHash string, peer,
 }
 
 func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, infoHash string, trackers []string) (int, int, error) {
-	var leech, seed int
 	leech, seed, err := getPeersFromCache(ctx, r, infoHash)
 	if err != nil {
 		fmt.Println("unable to get peers from cache for infohash:", infoHash)
@@ -49,27 +53,44 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, infoHash string, tra
 		return leech, seed, nil
 	}
 
+	var peerChan = make(chan peers)
+	var errChan = make(chan error)
+
 	for _, tracker := range trackers {
-		// get peers and seeds from redis first
+		go func(tracker string) {
-		scraper, err := New(tracker)
+			// get peers and seeds from redis first
-		if err != nil {
+			scraper, err := New(tracker)
-			fmt.Println(err)
+			if err != nil {
-			continue
+				errChan <- err
-		}
+				return
+			}
 
-		scraper.SetTimeout(500 * time.Millisecond)
+			scraper.SetTimeout(500 * time.Millisecond)
 
-		// get peers and seeds from redis first
+			// get peers and seeds from redis first
-		res, err := scraper.Scrape([]byte(infoHash))
+			res, err := scraper.Scrape([]byte(infoHash))
-		if err != nil {
+			if err != nil {
-			fmt.Println(err)
+				errChan <- err
-			continue
+				return
-		}
+			}
 
-		leech += int(res[0].Leechers)
+			peerChan <- peers{
-		seed += int(res[0].Seeders)
+				Seeders:  int(res[0].Seeders),
-		setPeersToCache(ctx, r, infoHash, leech, seed)
+				Leechers: int(res[0].Leechers),
-		return leech, seed, nil
+			}
+		}(tracker)
 	}
-	return leech, seed, nil
+
+	var peer peers
+	for i := 0; i < len(trackers); i++ {
+		select {
+		case peer = <-peerChan:
+			setPeersToCache(ctx, r, infoHash, peer.Leechers, peer.Seeders)
+			return peer.Leechers, peer.Seeders, nil
+		case err := <-errChan:
+			fmt.Println(err)
+		}
+	}
+
+	return 0, 0, fmt.Errorf("unable to get peers from trackers")
 }