chg: feat: add more audio options (#22)

* chg: fix: add pt-bt to audio detection

* chg: fix: add retry when flaresolverr response is without body

* chg: fix: add back under attack verification

README.md

@@ -10,6 +10,7 @@ Visit [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org
 
 - [comando-torrents](https://comando.la/)
 - [bludv](https://bludvfilmes.tv/)
+- [torrent-dos-filmes](https://torrentdosfilmes.se/)
 
 ## Deploy
 

api/bludv.go

@@ -44,7 +44,7 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 	}
 
 	fmt.Println("URL:>", url)
-	resp, err := http.Get(url)
+	resp, err := i.requester.GetDocument(ctx, url)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -54,9 +54,9 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 		i.metrics.IndexerErrors.WithLabelValues("bludv").Inc()
 		return
 	}
-	defer resp.Body.Close()
+	defer resp.Close()
 
-	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	doc, err := goquery.NewDocumentFromReader(resp)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})

api/comando_torrents.go

@@ -60,7 +60,7 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 	}
 
 	fmt.Println("URL:>", url)
-	resp, err := http.Get(url)
+	resp, err := i.requester.GetDocument(ctx, url)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -70,9 +70,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 		i.metrics.IndexerErrors.WithLabelValues("comando").Inc()
 		return
 	}
-	defer resp.Body.Close()
+	defer resp.Close()
 
-	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	doc, err := goquery.NewDocumentFromReader(resp)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -405,17 +405,18 @@ func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Documen
 	docCache, err := i.redis.Get(ctx, link)
 	if err == nil {
 		i.metrics.CacheHits.WithLabelValues("document_body").Inc()
+		fmt.Printf("returning from long-lived cache: %s\n", link)
 		return goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(docCache)))
 	}
 	defer i.metrics.CacheMisses.WithLabelValues("document_body").Inc()
 
-	resp, err := http.Get(link)
+	resp, err := i.requester.GetDocument(ctx, link)
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Body.Close()
+	defer resp.Close()
 
-	body, err := io.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp)
 	if err != nil {
 		return nil, err
 	}

api/index.go

@@ -7,12 +7,14 @@ import (
 
 	"github.com/felipemarinho97/torrent-indexer/cache"
 	"github.com/felipemarinho97/torrent-indexer/monitoring"
+	"github.com/felipemarinho97/torrent-indexer/requester"
 	"github.com/felipemarinho97/torrent-indexer/schema"
 )
 
 type Indexer struct {
-	redis   *cache.Redis
+	redis     *cache.Redis
-	metrics *monitoring.Metrics
+	metrics   *monitoring.Metrics
+	requester *requester.Requster
 }
 
 type IndexerMeta struct {
@@ -42,10 +44,11 @@ type IndexedTorrent struct {
 	Similarity    float32        `json:"similarity"`
 }
 
-func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics) *Indexer {
+func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster) *Indexer {
 	return &Indexer{
-		redis:   redis,
+		redis:     redis,
-		metrics: metrics,
+		metrics:   metrics,
+		requester: req,
 	}
 }
 
@@ -75,6 +78,16 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
 						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
 					}},
 			},
+			"/indexers/torrent-dos-filmes": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"description": "Indexer for Torrent dos Filmes",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
+				},
+			},
 			"/indexers/manual": []map[string]interface{}{
 				{
 					"method":      "POST",

api/torrent_dos_filmes.go

@@ -0,0 +1,267 @@
+package handler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/hbollon/go-edlib"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+var torrent_dos_filmes = IndexerMeta{
+	URL:       "https://torrentdosfilmes.se/",
+	SearchURL: "?s=",
+}
+
+func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues("torrent_dos_filmes").Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues("torrent_dos_filmes").Inc()
+	}()
+
+	ctx := r.Context()
+	// supported query params: q, season, episode, filter_results
+	q := r.URL.Query().Get("q")
+
+	// URL encode query param
+	q = url.QueryEscape(q)
+	url := torrent_dos_filmes.URL
+	if q != "" {
+		url = fmt.Sprintf("%s%s%s", url, torrent_dos_filmes.SearchURL, q)
+	}
+
+	fmt.Println("URL:>", url)
+	resp, err := i.requester.GetDocument(ctx, url)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues("torrent_dos_filmes").Inc()
+		return
+	}
+	defer resp.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		i.metrics.IndexerErrors.WithLabelValues("torrent_dos_filmes").Inc()
+		return
+	}
+
+	var links []string
+	doc.Find(".post").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Find("div.title > a").Attr("href")
+		links = append(links, link)
+	})
+
+	var itChan = make(chan []IndexedTorrent)
+	var errChan = make(chan error)
+	indexedTorrents := []IndexedTorrent{}
+	for _, link := range links {
+		go func(link string) {
+			torrents, err := getTorrentsTorrentDosFilmes(ctx, i, link)
+			if err != nil {
+				fmt.Println(err)
+				errChan <- err
+			}
+			itChan <- torrents
+		}(link)
+	}
+
+	for i := 0; i < len(links); i++ {
+		select {
+		case torrents := <-itChan:
+			indexedTorrents = append(indexedTorrents, torrents...)
+		case err := <-errChan:
+			fmt.Println(err)
+		}
+	}
+
+	for i, it := range indexedTorrents {
+		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
+		qLower := strings.ToLower(q)
+		splitLength := 2
+		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
+	}
+
+	// remove the ones with zero similarity
+	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
+		indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
+			return it.Similarity > 0
+		})
+	}
+
+	// sort by similarity
+	slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
+		return int((j.Similarity - i.Similarity) * 1000)
+	})
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: indexedTorrents,
+		Count:   len(indexedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
+func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
+	var indexedTorrents []IndexedTorrent
+	doc, err := getDocument(ctx, i, link)
+	if err != nil {
+		return nil, err
+	}
+
+	article := doc.Find("article")
+	title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1)
+	textContent := article.Find("div.content")
+	date := getPublishedDateTDF(doc)
+	magnets := textContent.Find("a[href^=\"magnet\"]")
+	var magnetLinks []string
+	magnets.Each(func(i int, s *goquery.Selection) {
+		magnetLink, _ := s.Attr("href")
+		magnetLinks = append(magnetLinks, magnetLink)
+	})
+
+	var audio []schema.Audio
+	var year string
+	var size []string
+	article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
+		// pattern:
+		// Título Traduzido: Fundação
+		// Título Original: Foundation
+		// IMDb: 7,5
+		// Ano de Lançamento: 2023
+		// Gênero: Ação | Aventura | Ficção
+		// Formato: MKV
+		// Qualidade: WEB-DL
+		// Áudio: Português | Inglês
+		// Idioma: Português | Inglês
+		// Legenda: Português
+		// Tamanho: –
+		// Qualidade de Áudio: 10
+		// Qualidade de Vídeo: 10
+		// Duração: 59 Min.
+		// Servidor: Torrent
+		text := s.Text()
+
+		audio = append(audio, findAudioFromText(text)...)
+		y := findYearFromText(text, title)
+		if y != "" {
+			year = y
+		}
+		size = append(size, findSizesFromText(text)...)
+	})
+
+	// find any link from imdb
+	imdbLink := ""
+	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Attr("href")
+		re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
+		matches := re.FindStringSubmatch(link)
+		if len(matches) > 0 {
+			imdbLink = matches[0]
+		}
+	})
+
+	size = stableUniq(size)
+
+	var chanIndexedTorrent = make(chan IndexedTorrent)
+
+	// for each magnet link, create a new indexed torrent
+	for it, magnetLink := range magnetLinks {
+		it := it
+		go func(it int, magnetLink string) {
+			magnet, err := magnet.ParseMagnetUri(magnetLink)
+			if err != nil {
+				fmt.Println(err)
+			}
+			releaseTitle := magnet.DisplayName
+			infoHash := magnet.InfoHash.String()
+			trackers := magnet.Trackers
+			magnetAudio := []schema.Audio{}
+			if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
+				magnetAudio = append(magnetAudio, audio...)
+			} else if len(audio) > 1 {
+				// remove portuguese audio, and append to magnetAudio
+				for _, a := range audio {
+					if a != schema.AudioPortuguese {
+						magnetAudio = append(magnetAudio, a)
+					}
+				}
+			} else {
+				magnetAudio = append(magnetAudio, audio...)
+			}
+
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+			if err != nil {
+				fmt.Println(err)
+			}
+
+			title := processTitle(title, magnetAudio)
+
+			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
+			var mySize string
+			if len(size) == len(magnetLinks) {
+				mySize = size[it]
+			}
+
+			ixt := IndexedTorrent{
+				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				OriginalTitle: title,
+				Details:       link,
+				Year:          year,
+				IMDB:          imdbLink,
+				Audio:         magnetAudio,
+				MagnetLink:    magnetLink,
+				Date:          date,
+				InfoHash:      infoHash,
+				Trackers:      trackers,
+				LeechCount:    peer,
+				SeedCount:     seed,
+				Size:          mySize,
+			}
+			chanIndexedTorrent <- ixt
+		}(it, magnetLink)
+	}
+
+	for i := 0; i < len(magnetLinks); i++ {
+		it := <-chanIndexedTorrent
+		indexedTorrents = append(indexedTorrents, it)
+	}
+
+	return indexedTorrents, nil
+}
+
+func getPublishedDateTDF(document *goquery.Document) time.Time {
+	var date time.Time
+	//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">
+	datePublished := strings.TrimSpace(document.Find("meta[property=\"article:published_time\"]").AttrOr("content", ""))
+
+	if datePublished != "" {
+		date, _ = time.Parse(time.RFC3339, datePublished)
+	}
+
+	return date
+}

cache/redis.go

@@ -10,7 +10,7 @@ import (
 )
 
 var (
-	DefaultExpiration      = 24 * time.Hour * 180 // 180 days
+	DefaultExpiration      = 24 * time.Hour * 7 // 7 days
 	IndexerComandoTorrents = "indexer:comando_torrents"
 )
 

docker-compose.yml

@@ -11,6 +11,7 @@ services:
       - indexer
     environment:
       - REDIS_HOST=redis
+      - FLARESOLVERR_ADDRESS=http://flaresolverr:8191
   
   redis:
     image: redis:alpine

main.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"fmt"
 	"net/http"
+	"os"
 
 	handler "github.com/felipemarinho97/torrent-indexer/api"
 	"github.com/felipemarinho97/torrent-indexer/cache"
 	"github.com/felipemarinho97/torrent-indexer/monitoring"
+	"github.com/felipemarinho97/torrent-indexer/requester"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -13,13 +16,17 @@ func main() {
 	redis := cache.NewRedis()
 	metrics := monitoring.NewMetrics()
 	metrics.Register()
-	indexers := handler.NewIndexers(redis, metrics)
+
+	flaresolverr := requester.NewFlareSolverr(os.Getenv("FLARESOLVERR_ADDRESS"), 60000)
+	req := requester.NewRequester(flaresolverr, redis)
+	indexers := handler.NewIndexers(redis, metrics, req)
 
 	indexerMux := http.NewServeMux()
 	metricsMux := http.NewServeMux()
 
 	indexerMux.HandleFunc("/", handler.HandlerIndex)
 	indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
+	indexerMux.HandleFunc("/indexers/torrent-dos-filmes", indexers.HandlerTorrentDosFilmesIndexer)
 	indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
 	indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer)
 
@@ -31,7 +38,7 @@ func main() {
 			panic(err)
 		}
 	}()
-
+	fmt.Println("Server listening on :7006")
 	err := http.ListenAndServe(":7006", indexerMux)
 	if err != nil {
 		panic(err)

requester/flaresolverr.go

@@ -0,0 +1,282 @@
+package requester
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"strings"
+	"sync"
+)
+
+type FlareSolverr struct {
+	url         string
+	maxTimeout  int
+	httpClient  *http.Client
+	sessionPool chan string
+	mu          sync.Mutex
+	initiated   bool
+}
+
+var (
+	ErrListSessions = fmt.Errorf("failed to list sessions")
+)
+
+func NewFlareSolverr(url string, timeoutMilli int) *FlareSolverr {
+	poolSize := 5
+	httpClient := &http.Client{}
+	sessionPool := make(chan string, poolSize) // Pool size of 5 sessions
+
+	f := &FlareSolverr{
+		url:         url,
+		maxTimeout:  timeoutMilli,
+		httpClient:  httpClient,
+		sessionPool: sessionPool,
+	}
+
+	err := f.FillSessionPool()
+	if err == nil {
+		f.initiated = true
+	}
+
+	return f
+}
+
+func (f *FlareSolverr) FillSessionPool() error {
+	// Check if the pool is already filled
+	if len(f.sessionPool) == cap(f.sessionPool) {
+		return nil
+	}
+
+	// Pre-initialize the pool with existing sessions
+	sessions, err := f.ListSessions()
+	if err != nil {
+		// if fail to list sessions, it may not support the sessions.list command
+		// create new dumb sessions to fill the pool
+		if err == ErrListSessions {
+			for len(f.sessionPool) < cap(f.sessionPool) {
+				f.sessionPool <- "dumb-session"
+			}
+			return nil
+		}
+		fmt.Println("Failed to list existing FlareSolverr sessions:", err)
+		return err
+	} else {
+		for _, session := range sessions {
+			// Add available sessions to the pool
+			if len(f.sessionPool) < cap(f.sessionPool) {
+				f.sessionPool <- session
+			}
+		}
+		if len(f.sessionPool) > 0 {
+			fmt.Printf("Added %d FlareSolverr sessions to the pool\n", len(f.sessionPool))
+		}
+	}
+
+	// If fewer than poolSize sessions were found, create new ones to fill the pool
+	for len(f.sessionPool) < cap(f.sessionPool) {
+		f.CreateSession()
+	}
+
+	return nil
+}
+
+func (f *FlareSolverr) CreateSession() string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	body := map[string]string{"cmd": "sessions.create"}
+	jsonBody, err := json.Marshal(body)
+	if err != nil {
+		return ""
+	}
+
+	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
+	if err != nil {
+		return ""
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := f.httpClient.Do(req)
+	if err != nil {
+		return ""
+	}
+
+	defer resp.Body.Close()
+
+	var sessionResponse map[string]interface{}
+	err = json.NewDecoder(resp.Body).Decode(&sessionResponse)
+	if err != nil {
+		return ""
+	}
+
+	session := sessionResponse["session"].(string)
+	// Add session to the pool
+	f.sessionPool <- session
+
+	fmt.Println("Created new FlareSolverr session:", session)
+	return session
+}
+
+func (f *FlareSolverr) ListSessions() ([]string, error) {
+	body := map[string]string{"cmd": "sessions.list"}
+	jsonBody, err := json.Marshal(body)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := f.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	var sessionsResponse map[string]interface{}
+	err = json.NewDecoder(resp.Body).Decode(&sessionsResponse)
+	if err != nil {
+		return nil, err
+	}
+	if sessionsResponse["sessions"] == nil {
+		return nil, ErrListSessions
+	}
+
+	sessions := sessionsResponse["sessions"].([]interface{})
+	var sessionIDs []string
+	for _, session := range sessions {
+		sessionIDs = append(sessionIDs, session.(string))
+	}
+
+	return sessionIDs, nil
+}
+
+func (f *FlareSolverr) RetrieveSession() string {
+	// Blocking receive from the session pool.
+	session := <-f.sessionPool
+	return session
+}
+
+type Response struct {
+	Status   string `json:"status"`
+	Message  string `json:"message"`
+	Solution struct {
+		Url     string `json:"url"`
+		Status  int    `json:"status"`
+		Cookies []struct {
+			Domain   string `json:"domain"`
+			Expiry   int    `json:"expiry"`
+			HttpOnly bool   `json:"httpOnly"`
+			Name     string `json:"name"`
+			Path     string `json:"path"`
+			SameSite string `json:"sameSite"`
+			Secure   bool   `json:"secure"`
+			Value    string `json:"value"`
+		} `json:"cookies"`
+		UserAgent string            `json:"userAgent"`
+		Headers   map[string]string `json:"headers"`
+		Response  string            `json:"response"`
+	} `json:"solution"`
+}
+
+func (f *FlareSolverr) Get(_url string) (io.ReadCloser, error) {
+	// Check if the FlareSolverr instance was initiated
+	if !f.initiated {
+		return io.NopCloser(bytes.NewReader([]byte(""))), nil
+	}
+
+	// Retrieve session from the pool (blocking if no sessions available)
+	session := f.RetrieveSession()
+
+	// Ensure the session is returned to the pool after the request is done
+	defer func() {
+		f.sessionPool <- session
+	}()
+
+	body := map[string]string{
+		"cmd":        "request.get",
+		"url":        _url,
+		"maxTimeout": fmt.Sprintf("%d", f.maxTimeout),
+		"session":    session,
+	}
+	jsonBody, err := json.Marshal(body)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := f.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse the response
+	var response Response
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if the response was successful
+	if response.Status != "ok" {
+		return nil, fmt.Errorf("failed to get response: %s", response.Message)
+	}
+
+	// Check if "Under attack" is in the response
+	if strings.Contains(response.Solution.Response, "Under attack") {
+		return nil, fmt.Errorf("under attack")
+	}
+
+	// If the response body is empty but cookies are present, make a new request
+	if response.Solution.Response == "" && len(response.Solution.Cookies) > 0 {
+		// Create a new request with cookies
+		client := &http.Client{}
+		cookieJar, err := cookiejar.New(&cookiejar.Options{})
+		if err != nil {
+			return nil, err
+		}
+		for _, cookie := range response.Solution.Cookies {
+			cookieJar.SetCookies(&url.URL{Host: cookie.Domain}, []*http.Cookie{
+				{
+					Name:   cookie.Name,
+					Value:  cookie.Value,
+					Domain: cookie.Domain,
+					Path:   cookie.Path,
+				},
+			})
+		}
+		client.Jar = cookieJar
+
+		secondReq, err := http.NewRequest("GET", _url, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		secondResp, err := client.Do(secondReq)
+		if err != nil {
+			return nil, err
+		}
+
+		// Return the body of the second request
+		return secondResp.Body, nil
+	}
+
+	// Return the original response body
+	return io.NopCloser(bytes.NewReader([]byte(response.Solution.Response))), nil
+}

requester/requester.go

@@ -0,0 +1,93 @@
+package requester
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"time"
+
+	"github.com/felipemarinho97/torrent-indexer/cache"
+)
+
+const (
+	shortLivedCacheExpiration = 30 * time.Minute
+	cacheKey                  = "shortLivedCache"
+)
+
+var challangeRegex = regexp.MustCompile(`(?i)(just a moment|cf-chl-bypass|under attack)`)
+
+type Requster struct {
+	fs         *FlareSolverr
+	c          *cache.Redis
+	httpClient *http.Client
+}
+
+func NewRequester(fs *FlareSolverr, c *cache.Redis) *Requster {
+	return &Requster{fs: fs, httpClient: &http.Client{}, c: c}
+}
+
+func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser, error) {
+	var body io.ReadCloser
+
+	// try request from short-lived cache
+	key := fmt.Sprintf("%s:%s", cacheKey, url)
+	bodyByte, err := i.c.Get(ctx, key)
+	if err == nil {
+		fmt.Printf("returning from short-lived cache: %s\n", url)
+		body = io.NopCloser(bytes.NewReader(bodyByte))
+		return body, nil
+	}
+
+	// try request with plain client
+	resp, err := i.httpClient.Get(url)
+	if err != nil {
+		// try request with flare solverr
+		body, err = i.fs.Get(url)
+		if err != nil {
+			return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
+		}
+	} else {
+		defer resp.Body.Close()
+		body = resp.Body
+	}
+
+	bodyByte, err = io.ReadAll(body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+	if hasChallange(bodyByte) {
+		// try request with flare solverr
+		body, err = i.fs.Get(url)
+		if err != nil {
+			return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
+		}
+		bodyByte, err = io.ReadAll(body)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read response body: %w", err)
+		}
+		fmt.Printf("request served from flaresolverr: %s\n", url)
+	} else {
+		fmt.Printf("request served from plain client: %s\n", url)
+	}
+
+	// save response to cache if it's not a challange and body is not empty
+	if !hasChallange(bodyByte) && len(bodyByte) > 0 {
+		err = i.c.SetWithExpiration(ctx, key, bodyByte, shortLivedCacheExpiration)
+		if err != nil {
+			fmt.Printf("failed to save response to cache: %v\n", err)
+		}
+		fmt.Printf("saved to cache: %s\n", url)
+	} else {
+		return nil, fmt.Errorf("response is a challange")
+	}
+
+	return io.NopCloser(bytes.NewReader(bodyByte)), nil
+}
+
+// hasChallange checks if the body contains a challange by regex matching
+func hasChallange(body []byte) bool {
+	return challangeRegex.Match(body)
+}

schema/audio.go

@@ -1,10 +1,14 @@
 package schema
 
+import "strings"
+
 type Audio string
 
 const (
 	AudioPortuguese  = "Português"
 	AudioPortuguese2 = "Portugues"
+	AudioPortuguese3 = "PT-BR"
+	AudioPortuguese4 = "Dublado"
 	AudioEnglish     = "Inglês"
 	AudioEnglish2    = "Ingles"
 	AudioSpanish     = "Espanhol"
@@ -28,11 +32,14 @@ const (
 	AudioThai        = "Tailandês"
 	AudioThai2       = "Tailandes"
 	AudioTurkish     = "Turco"
+	AudioHindi       = "Hindi"
 )
 
 var AudioList = []Audio{
 	AudioPortuguese,
 	AudioPortuguese2,
+	AudioPortuguese3,
+	AudioPortuguese4,
 	AudioEnglish,
 	AudioEnglish2,
 	AudioSpanish,
@@ -56,27 +63,32 @@ var AudioList = []Audio{
 	AudioThai,
 	AudioThai2,
 	AudioTurkish,
+	AudioHindi,
 }
 
 func (a Audio) String() string {
-	return a.toISO639_2()
+	return a.toTag()
 }
 
 func GetAudioFromString(s string) *Audio {
 	for _, a := range AudioList {
-		if string(a) == s {
+		if strings.EqualFold(string(a), s) {
 			return &a
 		}
 	}
 	return nil
 }
 
-func (a Audio) toISO639_2() string {
+func (a Audio) toTag() string {
 	switch a {
 	case AudioPortuguese:
-		return "por"
+		return "brazilian"
 	case AudioPortuguese2:
-		return "por"
+		return "brazilian"
+	case AudioPortuguese3:
+		return "brazilian"
+	case AudioPortuguese4:
+		return "brazilian"
 	case AudioEnglish:
 		return "eng"
 	case AudioEnglish2:
@@ -123,6 +135,8 @@ func (a Audio) toISO639_2() string {
 		return "tha"
 	case AudioTurkish:
 		return "tur"
+	case AudioHindi:
+		return "hin"
 	default:
 		return ""
 	}

scrape/info.go

@@ -52,7 +52,7 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metric
 		fmt.Println("unable to get peers from cache for infohash:", infoHash)
 	} else {
 		m.CacheMisses.WithLabelValues("peers").Inc()
-		fmt.Println("get from cache> leech:", leech, "seed:", seed)
+		fmt.Println("hash:", infoHash, "get from cache -> leech:", leech, "seed:", seed)
 		return leech, seed, nil
 	}
 
@@ -87,16 +87,18 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metric
 	var peer peers
 	for i := 0; i < len(trackers); i++ {
 		select {
+		case <-errChan:
+			// discard error
 		case peer = <-peerChan:
 			err = setPeersToCache(ctx, r, infoHash, peer.Leechers, peer.Seeders)
 			if err != nil {
 				fmt.Println(err)
+			} else {
+				fmt.Println("hash:", infoHash, "get from tracker -> leech:", peer.Leechers, "seed:", peer.Seeders)
 			}
 			return peer.Leechers, peer.Seeders, nil
-		case err := <-errChan:
-			fmt.Println(err)
 		}
 	}
 
-	return 0, 0, fmt.Errorf("unable to get peers from trackers")
+	return 0, 0, fmt.Errorf("unable to get peers from trackers for infohash: %s", infoHash)
 }