Merge branch 'main' of https://github.com/felipemarinho97/torrent-indexer

.golangci.yml

@@ -1,4 +1,3 @@
-issues:
+run:
-  exclude-files:
+  skip-files:
-    - scrape.go
+    - scrape.go
-    - infohash.go

README.md

@@ -4,103 +4,9 @@ This is a simple torrent indexer that can be used to index torrents from HTML pa
 
 ## Test it
 
-Visit [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org/) to test it.
+Visit [https://vlambdas.oci.darklyn.online/](https://vlambdas.oci.darklyn.online/) to test it.
 
 ## Supported sites
 
 - [comando-torrents](https://comando.la/)
-- [bludv](https://bludvfilmes.tv/)
+- [bludv](https://bludvfilmes.tv/)
-
-## Deploy
-
-If you have Docker + docker-compose installed, you can deploy it using the following command:
-
-```bash
-curl -s https://raw.githubusercontent.com/felipemarinho97/torrent-indexer/main/docker-compose.yml > docker-compose.yml
-docker-compose up -d
-```
-
-The server will be available at [http://localhost:8080/](http://localhost:8080/).
-
-## Integrating with Jackett
-
-You can integrate this indexer with Jackett by adding a new Torznab custom indexer. Here is an example of how to do it for the `bludv` indexer:
-
-```yaml
----
-id: bludv_indexer
-name: BluDV Indexer
-description: "BluDV - Custom indexer on from torrent-indexer"
-language: pt-BR
-type: public
-encoding: UTF-8
-links:
-  - http://localhost:8080/
-
-caps:
-  categorymappings:
-    - { id: Movie, cat: Movies, desc: "Movies" }
-    - { id: TV, cat: TV, desc: "TV" }
-
-  modes:
-    search: [q]
-    tv-search: [q, season, ep]
-    movie-search: [q]
-  allowrawsearch: true
-
-settings: []
-
-search:
-  paths:
-    - path: "indexers/bludv?filter_results=true&q={{ .Keywords }}"
-      response:
-        type: json
-
-  keywordsfilters:
-    - name: tolower
-
-  rows:
-    selector: $.results
-    count:
-      selector: $.count
-
-  fields:
-    _id:
-      selector: title
-    download:
-      selector: magnet_link
-    title:
-      selector: title
-    description:
-      selector: original_title
-    details:
-      selector: details
-    infohash:
-      selector: info_hash
-    date:
-      selector: date
-    size:
-      selector: size
-    seeders:
-      selector: seed_count
-    leechers:
-      selector: leech_count
-    imdb:
-      selector: imdb
-    category_is_tv_show:
-      selector: title
-      filters:
-        - name: regexp
-          args: "\\b(S\\d+(?:E\\d+)?)\\b"
-    category:
-      text: "{{ if .Result.category_is_tv_show }}TV{{ else }}Movie{{ end }}"
-# json engine n/a
-```
-
-If you have more tips on how to integrate with other torrent API clients like Prowlarr, please open a PR.
-
-# Warning
-
-The instance running at [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org/) is my personal instance and it is not guaranteed to be up all the time. Also, for better availability, I recommend deploying your own instance because the Cloudflare protection may block requests from indexed sites if too many requests are made in a short period of time from the same IP.
-
-If I notice that the instance is being used a lot, I may block requests from Jackett to avoid overloading the server without prior notice.

api/bludv.go

@@ -44,7 +44,7 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 	}
 
 	fmt.Println("URL:>", url)
-	resp, err := i.requester.GetDocument(ctx, url)
+	resp, err := http.Get(url)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -54,9 +54,9 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 		i.metrics.IndexerErrors.WithLabelValues("bludv").Inc()
 		return
 	}
-	defer resp.Close()
+	defer resp.Body.Close()
 
-	doc, err := goquery.NewDocumentFromReader(resp)
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})

api/comando_torrents.go

@@ -60,7 +60,7 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 	}
 
 	fmt.Println("URL:>", url)
-	resp, err := i.requester.GetDocument(ctx, url)
+	resp, err := http.Get(url)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -70,9 +70,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 		i.metrics.IndexerErrors.WithLabelValues("comando").Inc()
 		return
 	}
-	defer resp.Close()
+	defer resp.Body.Close()
 
-	doc, err := goquery.NewDocumentFromReader(resp)
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -405,18 +405,17 @@ func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Documen
 	docCache, err := i.redis.Get(ctx, link)
 	if err == nil {
 		i.metrics.CacheHits.WithLabelValues("document_body").Inc()
-		fmt.Printf("returning from long-lived cache: %s\n", link)
 		return goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(docCache)))
 	}
 	defer i.metrics.CacheMisses.WithLabelValues("document_body").Inc()
 
-	resp, err := i.requester.GetDocument(ctx, link)
+	resp, err := http.Get(link)
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Close()
+	defer resp.Body.Close()
 
-	body, err := io.ReadAll(resp)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err
 	}

api/index.go

@@ -7,14 +7,12 @@ import (
 
 	"github.com/felipemarinho97/torrent-indexer/cache"
 	"github.com/felipemarinho97/torrent-indexer/monitoring"
-	"github.com/felipemarinho97/torrent-indexer/requester"
 	"github.com/felipemarinho97/torrent-indexer/schema"
 )
 
 type Indexer struct {
-	redis     *cache.Redis
+	redis   *cache.Redis
-	metrics   *monitoring.Metrics
+	metrics *monitoring.Metrics
-	requester *requester.Requster
 }
 
 type IndexerMeta struct {
@@ -44,11 +42,10 @@ type IndexedTorrent struct {
 	Similarity    float32        `json:"similarity"`
 }
 
-func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster) *Indexer {
+func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics) *Indexer {
 	return &Indexer{
-		redis:     redis,
+		redis:   redis,
-		metrics:   metrics,
+		metrics: metrics,
-		requester: req,
 	}
 }
 
@@ -59,35 +56,20 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
 	err := json.NewEncoder(w).Encode(map[string]interface{}{
 		"time": currentTime,
 		"endpoints": map[string]interface{}{
-			"/indexers/comando_torrents": []map[string]interface{}{
+			"/indexers/comando_torrents": map[string]interface{}{
-				{
+				"method":      "GET",
-					"method":      "GET",
+				"description": "Indexer for comando torrents",
-					"description": "Indexer for comando torrents",
+				"query_params": map[string]string{
-					"query_params": map[string]string{
+					"q":              "search query",
-						"q":              "search query",
+					"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
-						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
-					},
 				},
 			},
-			"/indexers/bludv": []map[string]interface{}{
+			"/indexers/bludv": map[string]interface{}{
-				{
+				"method":      "GET",
-					"method":      "GET",
+				"description": "Indexer for bludv",
-					"description": "Indexer for bludv",
+				"query_params": map[string]string{
-					"query_params": map[string]string{
+					"q":              "search query",
-						"q":              "search query",
+					"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
-						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
-					}},
-			},
-			"/indexers/manual": []map[string]interface{}{
-				{
-					"method":      "POST",
-					"description": "Add a manual torrent entry to the indexer for 12 hours",
-					"body": map[string]interface{}{
-						"magnetLink": "magnet link",
-					}},
-				{
-					"method":      "GET",
-					"description": "Get all manual torrents",
 				},
 			},
 		},

api/manual.go

@@ -1,138 +0,0 @@
-package handler
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/felipemarinho97/torrent-indexer/magnet"
-	"github.com/felipemarinho97/torrent-indexer/schema"
-	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
-	"github.com/redis/go-redis/v9"
-)
-
-const manualTorrentsRedisKey = "manual:torrents"
-
-var manualTorrentExpiration = 8 * time.Hour
-
-type ManualIndexerRequest struct {
-	MagnetLink string `json:"magnetLink"`
-}
-
-func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-	var req ManualIndexerRequest
-	indexedTorrents := []IndexedTorrent{}
-
-	// fetch from redis
-	out, err := i.redis.Get(ctx, manualTorrentsRedisKey)
-	if err != nil && !errors.Is(err, redis.Nil) {
-		w.WriteHeader(http.StatusInternalServerError)
-		fmt.Println(err)
-		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-		if err != nil {
-			fmt.Println(err)
-		}
-		i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
-		return
-	} else if errors.Is(err, redis.Nil) {
-		out = bytes.NewBufferString("[]").Bytes()
-	}
-
-	err = json.Unmarshal([]byte(out), &indexedTorrents)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-		if err != nil {
-			fmt.Println(err)
-		}
-		i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
-		return
-	}
-
-	// check if the request is a POST
-	if r.Method == http.MethodPost {
-		// decode the request body
-		err := json.NewDecoder(r.Body).Decode(&req)
-		if err != nil {
-			w.WriteHeader(http.StatusBadRequest)
-			err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-			if err != nil {
-				fmt.Println(err)
-			}
-			i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
-			return
-		}
-
-		magnet, err := magnet.ParseMagnetUri(req.MagnetLink)
-		if err != nil {
-			fmt.Println(err)
-		}
-		var audio []schema.Audio
-		releaseTitle := magnet.DisplayName
-		infoHash := magnet.InfoHash.String()
-		trackers := magnet.Trackers
-		magnetAudio := []schema.Audio{}
-		if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
-			magnetAudio = append(magnetAudio, audio...)
-		} else if len(audio) > 1 {
-			// remove portuguese audio, and append to magnetAudio
-			for _, a := range audio {
-				if a != schema.AudioPortuguese {
-					magnetAudio = append(magnetAudio, a)
-				}
-			}
-		} else {
-			magnetAudio = append(magnetAudio, audio...)
-		}
-
-		peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		title := processTitle(releaseTitle, magnetAudio)
-
-		ixt := IndexedTorrent{
-			Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
-			OriginalTitle: title,
-			Audio:         magnetAudio,
-			MagnetLink:    req.MagnetLink,
-			InfoHash:      infoHash,
-			Trackers:      trackers,
-			LeechCount:    peer,
-			SeedCount:     seed,
-		}
-
-		// write to redis
-		indexedTorrents = append(indexedTorrents, ixt)
-		out, err := json.Marshal(indexedTorrents)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		err = i.redis.SetWithExpiration(ctx, manualTorrentsRedisKey, out, manualTorrentExpiration)
-		if err != nil {
-			w.WriteHeader(http.StatusInternalServerError)
-			err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-			if err != nil {
-				fmt.Println(err)
-			}
-			i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
-			return
-		}
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
-	})
-	if err != nil {
-		fmt.Println(err)
-	}
-}

cache/redis.go

@@ -10,7 +10,7 @@ import (
 )
 
 var (
-	DefaultExpiration      = 24 * time.Hour * 7 // 7 days
+	DefaultExpiration      = 24 * time.Hour * 180 // 180 days
 	IndexerComandoTorrents = "indexer:comando_torrents"
 )
 

docker-compose.yml

@@ -1,8 +1,11 @@
-version: '3'
+version: '3.7'
 
 services:
   torrent-indexer:
-    image: felipemarinho97/torrent-indexer:latest
+    image:
+      build:
+        context: .
+        dockerfile: Dockerfile
     container_name: torrent-indexer
     restart: unless-stopped
     ports:
@@ -11,7 +14,6 @@ services:
       - indexer
     environment:
       - REDIS_HOST=redis
-      - FLARESOLVERR_ADDRESS=http://flaresolverr:8191
   
   redis:
     image: redis:alpine

main.go

@@ -1,14 +1,11 @@
 package main
 
 import (
-	"fmt"
 	"net/http"
-	"os"
 
 	handler "github.com/felipemarinho97/torrent-indexer/api"
 	"github.com/felipemarinho97/torrent-indexer/cache"
 	"github.com/felipemarinho97/torrent-indexer/monitoring"
-	"github.com/felipemarinho97/torrent-indexer/requester"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -16,10 +13,7 @@ func main() {
 	redis := cache.NewRedis()
 	metrics := monitoring.NewMetrics()
 	metrics.Register()
-
+	indexers := handler.NewIndexers(redis, metrics)
-	flaresolverr := requester.NewFlareSolverr(os.Getenv("FLARESOLVERR_ADDRESS"), 60000)
-	req := requester.NewRequester(flaresolverr, redis)
-	indexers := handler.NewIndexers(redis, metrics, req)
 
 	indexerMux := http.NewServeMux()
 	metricsMux := http.NewServeMux()
@@ -27,7 +21,6 @@ func main() {
 	indexerMux.HandleFunc("/", handler.HandlerIndex)
 	indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
 	indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
-	indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer)
 
 	metricsMux.Handle("/metrics", promhttp.Handler())
 
@@ -37,7 +30,7 @@ func main() {
 			panic(err)
 		}
 	}()
-	fmt.Println("Server listening on :7006")
+
 	err := http.ListenAndServe(":7006", indexerMux)
 	if err != nil {
 		panic(err)

requester/flaresolverr.go

@@ -1,230 +0,0 @@
-package requester
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-	"sync"
-)
-
-type FlareSolverr struct {
-	url         string
-	maxTimeout  int
-	httpClient  *http.Client
-	sessionPool chan string
-	mu          sync.Mutex
-	initiated   bool
-}
-
-func NewFlareSolverr(url string, timeoutMilli int) *FlareSolverr {
-	poolSize := 5
-	httpClient := &http.Client{}
-	sessionPool := make(chan string, poolSize) // Pool size of 5 sessions
-
-	f := &FlareSolverr{
-		url:         url,
-		maxTimeout:  timeoutMilli,
-		httpClient:  httpClient,
-		sessionPool: sessionPool,
-	}
-
-	err := f.FillSessionPool()
-	if err == nil {
-		f.initiated = true
-	}
-
-	return f
-}
-
-func (f *FlareSolverr) FillSessionPool() error {
-	// Check if the pool is already filled
-	if len(f.sessionPool) == cap(f.sessionPool) {
-		return nil
-	}
-
-	// Pre-initialize the pool with existing sessions
-	sessions, err := f.ListSessions()
-	if err != nil {
-		fmt.Println("Failed to list existing FlareSolverr sessions:", err)
-		return err
-	} else {
-		for _, session := range sessions {
-			// Add available sessions to the pool
-			if len(f.sessionPool) < cap(f.sessionPool) {
-				f.sessionPool <- session
-			}
-		}
-		if len(f.sessionPool) > 0 {
-			fmt.Printf("Added %d FlareSolverr sessions to the pool\n", len(f.sessionPool))
-		}
-	}
-
-	// If fewer than poolSize sessions were found, create new ones to fill the pool
-	for len(f.sessionPool) < cap(f.sessionPool) {
-		f.CreateSession()
-	}
-
-	return nil
-}
-
-func (f *FlareSolverr) CreateSession() string {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-
-	body := map[string]string{"cmd": "sessions.create"}
-	jsonBody, err := json.Marshal(body)
-	if err != nil {
-		return ""
-	}
-
-	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
-	if err != nil {
-		return ""
-	}
-
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := f.httpClient.Do(req)
-	if err != nil {
-		return ""
-	}
-
-	defer resp.Body.Close()
-
-	var sessionResponse map[string]interface{}
-	err = json.NewDecoder(resp.Body).Decode(&sessionResponse)
-	if err != nil {
-		return ""
-	}
-
-	session := sessionResponse["session"].(string)
-	// Add session to the pool
-	f.sessionPool <- session
-
-	fmt.Println("Created new FlareSolverr session:", session)
-	return session
-}
-
-func (f *FlareSolverr) ListSessions() ([]string, error) {
-	body := map[string]string{"cmd": "sessions.list"}
-	jsonBody, err := json.Marshal(body)
-	if err != nil {
-		return nil, err
-	}
-
-	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := f.httpClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-
-	defer resp.Body.Close()
-
-	var sessionsResponse map[string]interface{}
-	err = json.NewDecoder(resp.Body).Decode(&sessionsResponse)
-	if err != nil {
-		return nil, err
-	}
-
-	sessions := sessionsResponse["sessions"].([]interface{})
-	var sessionIDs []string
-	for _, session := range sessions {
-		sessionIDs = append(sessionIDs, session.(string))
-	}
-
-	return sessionIDs, nil
-}
-
-func (f *FlareSolverr) RetrieveSession() string {
-	// Blocking receive from the session pool.
-	session := <-f.sessionPool
-	return session
-}
-
-type Response struct {
-	Status   string `json:"status"`
-	Message  string `json:"message"`
-	Solution struct {
-		Url     string `json:"url"`
-		Status  int    `json:"status"`
-		Cookies []struct {
-			Domain   string `json:"domain"`
-			Expiry   int    `json:"expiry"`
-			HttpOnly bool   `json:"httpOnly"`
-			Name     string `json:"name"`
-			Path     string `json:"path"`
-			SameSite string `json:"sameSite"`
-			Secure   bool   `json:"secure"`
-			Value    string `json:"value"`
-		} `json:"cookies"`
-		UserAgent string            `json:"userAgent"`
-		Headers   map[string]string `json:"headers"`
-		Response  string            `json:"response"`
-	} `json:"solution"`
-}
-
-func (f *FlareSolverr) Get(url string) (io.ReadCloser, error) {
-	// Check if the FlareSolverr instance was initiated
-	if !f.initiated {
-		return io.NopCloser(bytes.NewReader([]byte(""))), nil
-	}
-
-	// Retrieve session from the pool (blocking if no sessions available)
-	session := f.RetrieveSession()
-
-	// Ensure the session is returned to the pool after the request is done
-	defer func() {
-		f.sessionPool <- session
-	}()
-
-	body := map[string]string{
-		"cmd":        "request.get",
-		"url":        url,
-		"maxTimeout": fmt.Sprintf("%d", f.maxTimeout),
-		"session":    session,
-	}
-	jsonBody, err := json.Marshal(body)
-	if err != nil {
-		return nil, err
-	}
-	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := f.httpClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-
-	// Parse the response
-	var response Response
-	err = json.NewDecoder(resp.Body).Decode(&response)
-	if err != nil {
-		return nil, err
-	}
-
-	// Check if the response was successful
-	if response.Status != "ok" {
-		return nil, fmt.Errorf("failed to get response: %s", response.Message)
-	}
-
-	// Check if "Under attack" is in the response
-	if strings.Contains(response.Solution.Response, "Under attack") {
-		return nil, fmt.Errorf("under attack")
-	}
-
-	// Return the response body
-	return io.NopCloser(bytes.NewReader([]byte(response.Solution.Response))), nil
-}

requester/requester.go

@@ -1,93 +0,0 @@
-package requester
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"regexp"
-	"time"
-
-	"github.com/felipemarinho97/torrent-indexer/cache"
-)
-
-const (
-	shortLivedCacheExpiration = 30 * time.Minute
-	cacheKey                  = "shortLivedCache"
-)
-
-var challangeRegex = regexp.MustCompile(`(?i)(just a moment|cf-chl-bypass|under attack)`)
-
-type Requster struct {
-	fs         *FlareSolverr
-	c          *cache.Redis
-	httpClient *http.Client
-}
-
-func NewRequester(fs *FlareSolverr, c *cache.Redis) *Requster {
-	return &Requster{fs: fs, httpClient: &http.Client{}, c: c}
-}
-
-func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser, error) {
-	var body io.ReadCloser
-
-	// try request from short-lived cache
-	key := fmt.Sprintf("%s:%s", cacheKey, url)
-	bodyByte, err := i.c.Get(ctx, key)
-	if err == nil {
-		fmt.Printf("returning from short-lived cache: %s\n", url)
-		body = io.NopCloser(bytes.NewReader(bodyByte))
-		return body, nil
-	}
-
-	// try request with plain client
-	resp, err := i.httpClient.Get(url)
-	if err != nil {
-		// try request with flare solverr
-		body, err = i.fs.Get(url)
-		if err != nil {
-			return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
-		}
-	} else {
-		defer resp.Body.Close()
-		body = resp.Body
-	}
-
-	bodyByte, err = io.ReadAll(body)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read response body: %w", err)
-	}
-	if hasChallange(bodyByte) {
-		// try request with flare solverr
-		body, err = i.fs.Get(url)
-		if err != nil {
-			return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
-		}
-		bodyByte, err = io.ReadAll(body)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read response body: %w", err)
-		}
-		fmt.Printf("request served from flaresolverr: %s\n", url)
-	} else {
-		fmt.Printf("request served from plain client: %s\n", url)
-	}
-
-	// save response to cache if it's not a challange and body is not empty
-	if !hasChallange(bodyByte) && len(bodyByte) > 0 {
-		err = i.c.SetWithExpiration(ctx, key, bodyByte, shortLivedCacheExpiration)
-		if err != nil {
-			fmt.Printf("failed to save response to cache: %v\n", err)
-		}
-		fmt.Printf("saved to cache: %s\n", url)
-	} else {
-		return nil, fmt.Errorf("response is a challange")
-	}
-
-	return io.NopCloser(bytes.NewReader(bodyByte)), nil
-}
-
-// hasChallange checks if the body contains a challange by regex matching
-func hasChallange(body []byte) bool {
-	return challangeRegex.Match(body)
-}

scrape/info.go

@@ -52,7 +52,7 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metric
 		fmt.Println("unable to get peers from cache for infohash:", infoHash)
 	} else {
 		m.CacheMisses.WithLabelValues("peers").Inc()
-		fmt.Println("hash:", infoHash, "get from cache -> leech:", leech, "seed:", seed)
+		fmt.Println("get from cache> leech:", leech, "seed:", seed)
 		return leech, seed, nil
 	}
 
@@ -87,18 +87,16 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metric
 	var peer peers
 	for i := 0; i < len(trackers); i++ {
 		select {
-		case <-errChan:
-			// discard error
 		case peer = <-peerChan:
 			err = setPeersToCache(ctx, r, infoHash, peer.Leechers, peer.Seeders)
 			if err != nil {
 				fmt.Println(err)
-			} else {
-				fmt.Println("hash:", infoHash, "get from tracker -> leech:", peer.Leechers, "seed:", peer.Seeders)
 			}
 			return peer.Leechers, peer.Seeders, nil
+		case err := <-errChan:
+			fmt.Println(err)
 		}
 	}
 
-	return 0, 0, fmt.Errorf("unable to get peers from trackers for infohash: %s", infoHash)
+	return 0, 0, fmt.Errorf("unable to get peers from trackers")
 }