Merge branch 'main' of https://github.com/felipemarinho97/torrent-indexer

.golangci.yml

@@ -1,4 +1,3 @@
-issues:
+run:
-  exclude-files:
+  skip-files:
     - scrape.go
-    - infohash.go

README.md

@@ -4,205 +4,9 @@ This is a simple torrent indexer that can be used to index torrents from HTML pa
 
 ## Test it
 
-Visit [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org/) to test it.
+Visit [https://vlambdas.oci.darklyn.online/](https://vlambdas.oci.darklyn.online/) to test it.
 
 ## Supported sites
 
 - [comando-torrents](https://comando.la/)
 - [bludv](https://bludvfilmes.tv/)
-- [torrent-dos-filmes](https://torrentdosfilmes.se/)
-- [starck-filmes](https://www.starckfilmes.online/)
-- [comandohds](https://comandohds.org/)
-
-## Deploy
-
-If you have Docker + docker-compose installed, you can deploy it using the following command:
-
-```bash
-curl -s https://raw.githubusercontent.com/felipemarinho97/torrent-indexer/main/docker-compose.yml > docker-compose.yml
-docker-compose up -d
-```
-
-The server will be available at [http://localhost:8080/](http://localhost:8080/).
-
-## Configuration
-
-You can configure the server using the following environment variables:
-  
-- `PORT`: (optional) The port that the server will listen to. Default: `7006`
-- `FLARESOLVERR_ADDRESS`: (optional) The address of the FlareSolverr instance. Default: `N/A`
-- `MEILISEARCH_ADDRESS`: (optional) The address of the MeiliSearch instance. Default: `N/A`
-- `MEILISEARCH_KEY`: (optional) The API key of the MeiliSearch instance. Default: `N/A`
-- `REDIS_HOST`: (optional) The address of the Redis instance. Default: `localhost`
-- `SHORT_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the short-lived cache in duration format. Default: `30m`
-    - This cache is used to cache homepage or search results.
-    - Example: `30m`, `1h`, `1h30m`, `1h30m30s`
-- `LONG_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the long-lived cache in duration format. Default: `7d`
-    - This cache is used to store the torrent webpages (posts). You can set it to a higher value because the torrent pages are not updated frequently.
-
-## Integrating with Jackett
-
-You can integrate this indexer with Jackett by adding a new Torznab custom indexer. Here is an example of how to do it for the `bludv` indexer:
-
-```yaml
----
-id: bludv_indexer
-name: BluDV Indexer
-description: "BluDV - Custom indexer on from torrent-indexer"
-language: pt-BR
-type: public
-encoding: UTF-8
-links:
-  - http://localhost:8080/
-
-caps:
-  categorymappings:
-    - { id: Movie, cat: Movies, desc: "Movies" }
-    - { id: TV, cat: TV, desc: "TV" }
-
-  modes:
-    search: [q]
-    tv-search: [q, season, ep]
-    movie-search: [q]
-  allowrawsearch: true
-
-settings: []
-
-search:
-  paths:
-    - path: "indexers/bludv?filter_results=true&q={{ .Keywords }}"
-      response:
-        type: json
-
-  keywordsfilters:
-    - name: tolower
-
-  rows:
-    selector: $.results
-    count:
-      selector: $.count
-
-  fields:
-    _id:
-      selector: title
-    download:
-      selector: magnet_link
-    title:
-      selector: title
-    description:
-      selector: original_title
-    details:
-      selector: details
-    infohash:
-      selector: info_hash
-    date:
-      selector: date
-    size:
-      selector: size
-    seeders:
-      selector: seed_count
-    leechers:
-      selector: leech_count
-    imdb:
-      selector: imdb
-    category_is_tv_show:
-      selector: title
-      filters:
-        - name: regexp
-          args: "\\b(S\\d+(?:E\\d+)?)\\b"
-    category:
-      text: "{{ if .Result.category_is_tv_show }}TV{{ else }}Movie{{ end }}"
-# json engine n/a
-```
-
-## Integrating with Prowlarr
-
-You can integrate this indexer with Prowlarr by adding a custom definition. See [Adding a custom YML definition](https://wiki.servarr.com/prowlarr/indexers#adding-a-custom-yml-definition).
-
-```yaml
----
----
-id: torrent-indexer
-name: Torrent Indexer
-description: "Indexing Brazilian Torrent websites into structured data. github.com/felipemarinho97/torrent-indexer"
-language: pt-BR
-type: public
-encoding: UTF-8
-links:
-  - http://localhost:8080/
-
-caps:
-  categories:
-    Movies: Movies
-    TV: TV
-
-  modes:
-    search: [q]
-    tv-search: [q, season]
-    movie-search: [q]
-
-settings:
-  - name: indexer
-    type: select
-    label: Indexer
-    default: bludv
-    options:
-      bludv: BLUDV
-      comando_torrents: Comando Torrents
-      torrent-dos-filmes: Torrent dos Filmes
-
-search:
-  paths:
-    - path: "/indexers/{{ .Config.indexer }}"
-      response:
-        type: json
-  inputs:
-    filter_results: "true"
-    q: "{{ .Keywords }}"
-  keywordsfilters:
-    - name: tolower
-    - name: re_replace
-      args: ["(?i)(S0)(\\d{1,2})$", "temporada $2"]
-    - name: re_replace
-      args: ["(?i)(S)(\\d{1,3})$", "temporada $2"]
-
-  rows:
-    selector: $.results
-    count:
-      selector: $.count
-
-  fields:
-    download:
-      selector: magnet_link
-    title:
-      selector: title
-    description:
-      selector: original_title
-    details:
-      selector: details
-    infohash:
-      selector: info_hash
-    date:
-      selector: date
-    size:
-      selector: size
-    seeders:
-      selector: seed_count
-    leechers:
-      selector: leech_count
-    imdb:
-      selector: imdb
-    category_is_tv_show:
-      selector: title
-      filters:
-        - name: regexp
-          args: "\\b(S\\d+(?:E\\d+)?)\\b"
-    category:
-      text: "{{ if .Result.category_is_tv_show }}TV{{ else }}Movies{{ end }}"
-```
-
-# Warning
-
-The instance running at [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org/) is my personal instance and it is not guaranteed to be up all the time. Also, for better availability, I recommend deploying your own instance because the Cloudflare protection may block requests from indexed sites if too many requests are made in a short period of time from the same IP.
-
-If I notice that the instance is being used a lot, I may block requests from Jackett to avoid overloading the server without prior notice.

api/bludv.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"regexp"
 	"slices"
 	"strings"
 	"time"
@@ -20,7 +21,7 @@ import (
 )
 
 var bludv = IndexerMeta{
-	URL:       "https://bludv.xyz/",
+	URL:       "https://bludvfilmes.tv/",
 	SearchURL: "?s=",
 }
 
@@ -32,21 +33,18 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 	}()
 
 	ctx := r.Context()
-	// supported query params: q, season, episode, page, filter_results
+	// supported query params: q, season, episode, filter_results
 	q := r.URL.Query().Get("q")
-	page := r.URL.Query().Get("page")
 
 	// URL encode query param
 	q = url.QueryEscape(q)
 	url := bludv.URL
-	if page != "" {
+	if q != "" {
-		url = fmt.Sprintf("%spage/%s", url, page)
-	} else {
 		url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q)
 	}
 
 	fmt.Println("URL:>", url)
-	resp, err := i.requester.GetDocument(ctx, url)
+	resp, err := http.Get(url)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -56,9 +54,9 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 		i.metrics.IndexerErrors.WithLabelValues("bludv").Inc()
 		return
 	}
-	defer resp.Close()
+	defer resp.Body.Close()
 
-	doc, err := goquery.NewDocumentFromReader(resp)
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -77,9 +75,9 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []schema.IndexedTorrent)
+	var itChan = make(chan []IndexedTorrent)
 	var errChan = make(chan error)
-	indexedTorrents := []schema.IndexedTorrent{}
+	indexedTorrents := []IndexedTorrent{}
 	for _, link := range links {
 		go func(link string) {
 			torrents, err := getTorrentsBluDV(ctx, i, link)
@@ -109,21 +107,16 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 
 	// remove the ones with zero similarity
 	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
+		indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
 			return it.Similarity > 0
 		})
 	}
 
 	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
+	slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
 		return int((j.Similarity - i.Similarity) * 1000)
 	})
 
-	// send to search index
-	go func() {
-		_ = i.search.IndexTorrents(indexedTorrents)
-	}()
-
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(Response{
 		Results: indexedTorrents,
@@ -134,8 +127,8 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
-	var indexedTorrents []schema.IndexedTorrent
+	var indexedTorrents []IndexedTorrent
 	doc, err := getDocument(ctx, i, link)
 	if err != nil {
 		return nil, err
@@ -186,15 +179,16 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In
 	imdbLink := ""
 	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
 		link, _ := s.Attr("href")
-		_imdbLink, err := getIMDBLink(link)
+		re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
-		if err == nil {
+		matches := re.FindStringSubmatch(link)
-			imdbLink = _imdbLink
+		if len(matches) > 0 {
+			imdbLink = matches[0]
 		}
 	})
 
 	size = stableUniq(size)
 
-	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
+	var chanIndexedTorrent = make(chan IndexedTorrent)
 
 	// for each magnet link, create a new indexed torrent
 	for it, magnetLink := range magnetLinks {
@@ -234,7 +228,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In
 				mySize = size[it]
 			}
 
-			ixt := schema.IndexedTorrent{
+			ixt := IndexedTorrent{
 				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
 				OriginalTitle: title,
 				Details:       link,

api/comando_torrents.go

@@ -49,21 +49,18 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 	}()
 
 	ctx := r.Context()
-	// supported query params: q, season, episode, page, filter_results
+	// supported query params: q, season, episode
 	q := r.URL.Query().Get("q")
-	page := r.URL.Query().Get("page")
 
 	// URL encode query param
 	q = url.QueryEscape(q)
 	url := comando.URL
 	if q != "" {
 		url = fmt.Sprintf("%s%s%s", url, comando.SearchURL, q)
-	} else if page != "" {
-		url = fmt.Sprintf("%spage/%s", url, page)
 	}
 
 	fmt.Println("URL:>", url)
-	resp, err := i.requester.GetDocument(ctx, url)
+	resp, err := http.Get(url)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -73,9 +70,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 		i.metrics.IndexerErrors.WithLabelValues("comando").Inc()
 		return
 	}
-	defer resp.Close()
+	defer resp.Body.Close()
 
-	doc, err := goquery.NewDocumentFromReader(resp)
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -93,9 +90,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []schema.IndexedTorrent)
+	var itChan = make(chan []IndexedTorrent)
 	var errChan = make(chan error)
-	indexedTorrents := []schema.IndexedTorrent{}
+	indexedTorrents := []IndexedTorrent{}
 	for _, link := range links {
 		go func(link string) {
 			torrents, err := getTorrents(ctx, i, link)
@@ -125,21 +122,16 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 
 	// remove the ones with zero similarity
 	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
+		indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
 			return it.Similarity > 0
 		})
 	}
 
 	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
+	slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
 		return int((j.Similarity - i.Similarity) * 1000)
 	})
 
-	// send to search index
-	go func() {
-		_ = i.search.IndexTorrents(indexedTorrents)
-	}()
-
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(Response{
 		Results: indexedTorrents,
@@ -150,8 +142,8 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
-	var indexedTorrents []schema.IndexedTorrent
+	var indexedTorrents []IndexedTorrent
 	doc, err := getDocument(ctx, i, link)
 	if err != nil {
 		return nil, err
@@ -163,11 +155,19 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.Indexed
 	// div itemprop="datePublished"
 	datePublished := strings.TrimSpace(article.Find("div[itemprop=\"datePublished\"]").Text())
 	// pattern: 10 de setembro de 2021
-	date, err := parseLocalizedDate(datePublished)
+	re := regexp.MustCompile(`(\d{2}) de (\w+) de (\d{4})`)
+	matches := re.FindStringSubmatch(datePublished)
+	var date time.Time
+	if len(matches) > 0 {
+		day := matches[1]
+		month := matches[2]
+		year := matches[3]
+		datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
+		date, err = time.Parse("2006-01-02", datePublished)
 		if err != nil {
 			return nil, err
 		}
-
+	}
 	magnets := textContent.Find("a[href^=\"magnet\"]")
 	var magnetLinks []string
 	magnets.Each(func(i int, s *goquery.Selection) {
@@ -209,15 +209,16 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.Indexed
 	imdbLink := ""
 	article.Find("a").Each(func(i int, s *goquery.Selection) {
 		link, _ := s.Attr("href")
-		_imdbLink, err := getIMDBLink(link)
+		re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
-		if err == nil {
+		matches := re.FindStringSubmatch(link)
-			imdbLink = _imdbLink
+		if len(matches) > 0 {
+			imdbLink = matches[0]
 		}
 	})
 
 	size = stableUniq(size)
 
-	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
+	var chanIndexedTorrent = make(chan IndexedTorrent)
 
 	// for each magnet link, create a new indexed torrent
 	for it, magnetLink := range magnetLinks {
@@ -257,7 +258,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.Indexed
 				mySize = size[it]
 			}
 
-			ixt := schema.IndexedTorrent{
+			ixt := IndexedTorrent{
 				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
 				OriginalTitle: title,
 				Details:       link,
@@ -284,40 +285,6 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.Indexed
 	return indexedTorrents, nil
 }
 
-func getIMDBLink(link string) (string, error) {
-	var imdbLink string
-	re := regexp.MustCompile(`https://www.imdb.com(/[a-z]{2})?/title/(tt\d+)/?`)
-
-	matches := re.FindStringSubmatch(link)
-	if len(matches) > 0 {
-		imdbLink = matches[0]
-	} else {
-		return "", fmt.Errorf("no imdb link found")
-	}
-	return imdbLink, nil
-}
-
-func parseLocalizedDate(datePublished string) (time.Time, error) {
-	re := regexp.MustCompile(`(\d{1,2}) de (\w+) de (\d{4})`)
-	matches := re.FindStringSubmatch(datePublished)
-	if len(matches) > 0 {
-		day := matches[1]
-		// append 0 to single digit day
-		if len(day) == 1 {
-			day = fmt.Sprintf("0%s", day)
-		}
-		month := matches[2]
-		year := matches[3]
-		datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
-		date, err := time.Parse("2006-01-02", datePublished)
-		if err != nil {
-			return time.Time{}, err
-		}
-		return date, nil
-	}
-	return time.Time{}, nil
-}
-
 func stableUniq(s []string) []string {
 	var uniq []map[string]interface{}
 	m := make(map[string]map[string]interface{})
@@ -364,7 +331,7 @@ func findYearFromText(text string, title string) (year string) {
 			year = yearMatch[1]
 		}
 	}
-	return strings.TrimSpace(year)
+	return year
 }
 
 func findAudioFromText(text string) []schema.Audio {
@@ -438,18 +405,17 @@ func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Documen
 	docCache, err := i.redis.Get(ctx, link)
 	if err == nil {
 		i.metrics.CacheHits.WithLabelValues("document_body").Inc()
-		fmt.Printf("returning from long-lived cache: %s\n", link)
 		return goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(docCache)))
 	}
 	defer i.metrics.CacheMisses.WithLabelValues("document_body").Inc()
 
-	resp, err := i.requester.GetDocument(ctx, link)
+	resp, err := http.Get(link)
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Close()
+	defer resp.Body.Close()
 
-	body, err := io.ReadAll(resp)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err
 	}

api/comando_torrents_test.go

@@ -3,7 +3,6 @@ package handler
 import (
 	"reflect"
 	"testing"
-	"time"
 
 	"github.com/felipemarinho97/torrent-indexer/schema"
 )
@@ -78,115 +77,3 @@ Servidor Via: Torrent
 		})
 	}
 }
-
-func Test_parseLocalizedDate(t *testing.T) {
-	type args struct {
-		datePublished string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    time.Time
-		wantErr bool
-	}{
-		{
-			name: "should return date",
-			args: args{
-				datePublished: "12 de outubro de 2022",
-			},
-			want:    time.Date(2022, 10, 12, 0, 0, 0, 0, time.UTC),
-			wantErr: false,
-		},
-		{
-			name: "should return date single digit",
-			args: args{
-				datePublished: "1 de outubro de 2022",
-			},
-			want:    time.Date(2022, 10, 1, 0, 0, 0, 0, time.UTC),
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := parseLocalizedDate(tt.args.datePublished)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("parseDate() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("parseDate() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_getIMDBLink(t *testing.T) {
-	type args struct {
-		link string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    string
-		wantErr bool
-	}{
-		{
-			name: "should return imdb link",
-			args: args{
-				link: "https://www.imdb.com/title/tt1234567",
-			},
-			want:    "https://www.imdb.com/title/tt1234567",
-			wantErr: false,
-		},
-		{
-			name: "should return imdb link when end with /",
-			args: args{
-				link: "https://www.imdb.com/title/tt1234567/",
-			},
-			want:    "https://www.imdb.com/title/tt1234567/",
-			wantErr: false,
-		},
-		{
-			name: "should return imdb link when end with /",
-			args: args{
-				link: "https://www.imdb.com/title/tt1234567/",
-			},
-			want:    "https://www.imdb.com/title/tt1234567/",
-			wantErr: false,
-		},
-		{
-			name: "should return imdb link when it has a language",
-			args: args{
-				link: "https://www.imdb.com/pt/title/tt18722864/",
-			},
-			want: "https://www.imdb.com/pt/title/tt18722864/",
-		},
-		{
-			name: "should return imdb link when it has a language",
-			args: args{
-				link: "https://www.imdb.com/pt/title/tt34608980/",
-			},
-			want: "https://www.imdb.com/pt/title/tt34608980/",
-		},
-		{
-			name: "should return error when link is invalid",
-			args: args{
-				link: "https://www.google.com",
-			},
-			want:    "",
-			wantErr: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := getIMDBLink(tt.args.link)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("getIMDBLink() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if got != tt.want {
-				t.Errorf("getIMDBLink() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}

api/comandohds.go

@@ -1,263 +0,0 @@
-package handler
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"net/url"
-	"slices"
-	"strings"
-	"time"
-	"regexp"
-
-	"github.com/PuerkitoBio/goquery"
-	"github.com/hbollon/go-edlib"
-
-	"github.com/felipemarinho97/torrent-indexer/magnet"
-	"github.com/felipemarinho97/torrent-indexer/schema"
-	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
-	"github.com/felipemarinho97/torrent-indexer/utils"
-)
-
-var comandohds = IndexerMeta{
-	URL:       "https://comandohds.org/",
-	SearchURL: "?s=",
-}
-
-var title_re = regexp.MustCompile(`^[(Filme)|(Série)\s]+`)
-
-func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Request) {
-	start := time.Now()
-	defer func() {
-		i.metrics.IndexerDuration.WithLabelValues("comandohds").Observe(time.Since(start).Seconds())
-		i.metrics.IndexerRequests.WithLabelValues("comandohds").Inc()
-	}()
-
-	ctx := r.Context()
-	// supported query params: q, page, filter_results
-	q := r.URL.Query().Get("q")
-	page := r.URL.Query().Get("page")
-
-	// URL encode query param
-	q = url.QueryEscape(q)
-	url := comandohds.URL
-	if q != "" {
-		url = fmt.Sprintf("%s%s%s", url, comandohds.SearchURL, q)
-	} else if page != "" {
-		url = fmt.Sprintf("%spage/%s", url, page)
-	}
-
-	fmt.Println("URL:>", url)
-	resp, err := i.requester.GetDocument(ctx, url)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-		if err != nil {
-			fmt.Println(err)
-		}
-		i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
-		return
-	}
-	defer resp.Close()
-
-	doc, err := goquery.NewDocumentFromReader(resp)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
-		return
-	}
-
-	var links []string
-	doc.Find(".post").Each(func(i int, s *goquery.Selection) {
-		link, _ := s.Find("div.title > a").Attr("href")
-		links = append(links, link)
-	})
-
-	var itChan = make(chan []schema.IndexedTorrent)
-	var errChan = make(chan error)
-	indexedTorrents := []schema.IndexedTorrent{}
-	for _, link := range links {
-		go func(link string) {
-			torrents, err := getTorrentsComandoHDs(ctx, i, link)
-			if err != nil {
-				fmt.Println(err)
-				errChan <- err
-			}
-			itChan <- torrents
-		}(link)
-	}
-
-	for i := 0; i < len(links); i++ {
-		select {
-		case torrents := <-itChan:
-			indexedTorrents = append(indexedTorrents, torrents...)
-		case err := <-errChan:
-			fmt.Println(err)
-		}
-	}
-
-	for i, it := range indexedTorrents {
-		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
-		qLower := strings.ToLower(q)
-		splitLength := 2
-		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
-	}
-
-	// remove the ones with zero similarity
-	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
-			return it.Similarity > 0
-		})
-	}
-
-	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
-		return int((j.Similarity - i.Similarity) * 1000)
-	})
-
-	// send to search index
-	go func() {
-		_ = i.search.IndexTorrents(indexedTorrents)
-	}()
-
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
-	})
-	if err != nil {
-		fmt.Println(err)
-	}
-}
-
-func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
-	var indexedTorrents []schema.IndexedTorrent
-	doc, err := getDocument(ctx, i, link)
-	if err != nil {
-		return nil, err
-	}
-
-	article := doc.Find("article")
-	title := title_re.ReplaceAllString(article.Find(".main_title > h1").Text(), "")
-	textContent := article.Find("div.content")
-	date := getPublishedDateTDF(doc)
-	magnets := textContent.Find("a[href^=\"magnet\"]")
-	var magnetLinks []string
-	magnets.Each(func(i int, s *goquery.Selection) {
-		magnetLink, _ := s.Attr("href")
-		magnetLinks = append(magnetLinks, magnetLink)
-	})
-
-	var audio []schema.Audio
-	var year string
-	var size []string
-	article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
-		// pattern:
-		// »INFORMAÇÕES«
-		// Titulo Traduzido: O Guerreiro Banido
-		// Titulo Original: 天龍八部之喬峰傳
-		// <picture />: 5.7
-		// Ano de Lançamento: 2023
-		// Gênero: Ação
-		// Formato: MKV
-		// Qualidade: WEB-DL
-		// Idioma: Português | Inglês
-		// Legenda: Português
-		// Tamanho: – GB
-		// Qualidade Áudio e Vídeo: 10
-		// Duração: 130 Min
-		// Servidor: Torrent
-		text := s.Text()
-
-		audio = append(audio, findAudioFromText(text)...)
-		y := findYearFromText(text, title)
-		if y != "" {
-			year = y
-		}
-		size = append(size, findSizesFromText(text)...)
-	})
-
-	// find any link from imdb
-	imdbLink := ""
-	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
-		link, _ := s.Attr("href")
-		_imdbLink, err := getIMDBLink(link)
-		if err == nil {
-			imdbLink = _imdbLink
-		}
-	})
-
-	size = stableUniq(size)
-
-	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
-
-	// for each magnet link, create a new indexed torrent
-	for it, magnetLink := range magnetLinks {
-		it := it
-		go func(it int, magnetLink string) {
-			magnet, err := magnet.ParseMagnetUri(magnetLink)
-			if err != nil {
-				fmt.Println(err)
-			}
-			releaseTitle := magnet.DisplayName
-			infoHash := magnet.InfoHash.String()
-			trackers := magnet.Trackers
-			magnetAudio := []schema.Audio{}
-			if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
-				magnetAudio = append(magnetAudio, audio...)
-			} else if len(audio) > 1 {
-				// remove portuguese audio, and append to magnetAudio
-				for _, a := range audio {
-					if a != schema.AudioPortuguese {
-						magnetAudio = append(magnetAudio, a)
-					}
-				}
-			} else {
-				magnetAudio = append(magnetAudio, audio...)
-			}
-
-			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
-			if err != nil {
-				fmt.Println(err)
-			}
-
-			title := processTitle(title, magnetAudio)
-
-			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
-			var mySize string
-			if len(size) == len(magnetLinks) {
-				mySize = size[it]
-			}
-
-			ixt := schema.IndexedTorrent{
-				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
-				OriginalTitle: title,
-				Details:       link,
-				Year:          year,
-				IMDB:          imdbLink,
-				Audio:         magnetAudio,
-				MagnetLink:    magnetLink,
-				Date:          date,
-				InfoHash:      infoHash,
-				Trackers:      trackers,
-				LeechCount:    peer,
-				SeedCount:     seed,
-				Size:          mySize,
-			}
-			chanIndexedTorrent <- ixt
-		}(it, magnetLink)
-	}
-
-	for i := 0; i < len(magnetLinks); i++ {
-		it := <-chanIndexedTorrent
-		indexedTorrents = append(indexedTorrents, it)
-	}
-
-	return indexedTorrents, nil
-}

api/index.go

@@ -7,16 +7,12 @@ import (
 
 	"github.com/felipemarinho97/torrent-indexer/cache"
 	"github.com/felipemarinho97/torrent-indexer/monitoring"
-	"github.com/felipemarinho97/torrent-indexer/requester"
 	"github.com/felipemarinho97/torrent-indexer/schema"
-	meilisearch "github.com/felipemarinho97/torrent-indexer/search"
 )
 
 type Indexer struct {
 	redis   *cache.Redis
 	metrics *monitoring.Metrics
-	requester *requester.Requster
-	search    *meilisearch.SearchIndexer
 }
 
 type IndexerMeta struct {
@@ -25,16 +21,31 @@ type IndexerMeta struct {
 }
 
 type Response struct {
-	Results []schema.IndexedTorrent `json:"results"`
+	Results []IndexedTorrent `json:"results"`
 	Count   int              `json:"count"`
 }
 
-func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster, si *meilisearch.SearchIndexer) *Indexer {
+type IndexedTorrent struct {
+	Title         string         `json:"title"`
+	OriginalTitle string         `json:"original_title"`
+	Details       string         `json:"details"`
+	Year          string         `json:"year"`
+	IMDB          string         `json:"imdb"`
+	Audio         []schema.Audio `json:"audio"`
+	MagnetLink    string         `json:"magnet_link"`
+	Date          time.Time      `json:"date"`
+	InfoHash      string         `json:"info_hash"`
+	Trackers      []string       `json:"trackers"`
+	Size          string         `json:"size"`
+	LeechCount    int            `json:"leech_count"`
+	SeedCount     int            `json:"seed_count"`
+	Similarity    float32        `json:"similarity"`
+}
+
+func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics) *Indexer {
 	return &Indexer{
 		redis:   redis,
 		metrics: metrics,
-		requester: req,
-		search:    si,
 	}
 }
 
@@ -45,79 +56,20 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
 	err := json.NewEncoder(w).Encode(map[string]interface{}{
 		"time": currentTime,
 		"endpoints": map[string]interface{}{
-			"/indexers/comando_torrents": []map[string]interface{}{
+			"/indexers/comando_torrents": map[string]interface{}{
-				{
 				"method":      "GET",
 				"description": "Indexer for comando torrents",
 				"query_params": map[string]string{
 					"q":              "search query",
-						"page":           "page number",
 					"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
 				},
 			},
-			},
+			"/indexers/bludv": map[string]interface{}{
-			"/indexers/bludv": []map[string]interface{}{
-				{
 				"method":      "GET",
 				"description": "Indexer for bludv",
 				"query_params": map[string]string{
 					"q":              "search query",
-						"page":           "page number",
 					"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
-					}},
-			},
-			"/indexers/torrent-dos-filmes": []map[string]interface{}{
-				{
-					"method":      "GET",
-					"page":        "page number",
-					"description": "Indexer for Torrent dos Filmes",
-					"query_params": map[string]string{
-						"q":              "search query",
-						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
-					},
-				},
-			},
-			"/indexers/comandohds": []map[string]interface{}{
-				{
-					"method":      "GET",
-					"page":        "page number",
-					"description": "Indexer for Comando HDs",
-					"query_params": map[string]string{
-						"q":              "search query",
-						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
-					},
-				},
-			},
-			"/indexers/starck-filmes": []map[string]interface{}{
-				{
-					"method":      "GET",
-					"page":        "page number",
-					"description": "Indexer for Starck Filmes",
-					"query_params": map[string]string{
-						"q":              "search query",
-						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
-					},
-				},
-			},
-			"/indexers/manual": []map[string]interface{}{
-				{
-					"method":      "POST",
-					"description": "Add a manual torrent entry to the indexer for 12 hours",
-					"body": map[string]interface{}{
-						"magnetLink": "magnet link",
-					}},
-				{
-					"method":      "GET",
-					"description": "Get all manual torrents",
-				},
-			},
-			"/search": []map[string]interface{}{
-				{
-					"method":      "GET",
-					"description": "Search for cached torrents across all indexers",
-					"query_params": map[string]string{
-						"q": "search query",
-					},
 				},
 			},
 		},

api/manual.go

@@ -1,138 +0,0 @@
-package handler
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/felipemarinho97/torrent-indexer/magnet"
-	"github.com/felipemarinho97/torrent-indexer/schema"
-	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
-	"github.com/redis/go-redis/v9"
-)
-
-const manualTorrentsRedisKey = "manual:torrents"
-
-var manualTorrentExpiration = 8 * time.Hour
-
-type ManualIndexerRequest struct {
-	MagnetLink string `json:"magnetLink"`
-}
-
-func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-	var req ManualIndexerRequest
-	indexedTorrents := []schema.IndexedTorrent{}
-
-	// fetch from redis
-	out, err := i.redis.Get(ctx, manualTorrentsRedisKey)
-	if err != nil && !errors.Is(err, redis.Nil) {
-		w.WriteHeader(http.StatusInternalServerError)
-		fmt.Println(err)
-		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-		if err != nil {
-			fmt.Println(err)
-		}
-		i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
-		return
-	} else if errors.Is(err, redis.Nil) {
-		out = bytes.NewBufferString("[]").Bytes()
-	}
-
-	err = json.Unmarshal([]byte(out), &indexedTorrents)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-		if err != nil {
-			fmt.Println(err)
-		}
-		i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
-		return
-	}
-
-	// check if the request is a POST
-	if r.Method == http.MethodPost {
-		// decode the request body
-		err := json.NewDecoder(r.Body).Decode(&req)
-		if err != nil {
-			w.WriteHeader(http.StatusBadRequest)
-			err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-			if err != nil {
-				fmt.Println(err)
-			}
-			i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
-			return
-		}
-
-		magnet, err := magnet.ParseMagnetUri(req.MagnetLink)
-		if err != nil {
-			fmt.Println(err)
-		}
-		var audio []schema.Audio
-		releaseTitle := magnet.DisplayName
-		infoHash := magnet.InfoHash.String()
-		trackers := magnet.Trackers
-		magnetAudio := []schema.Audio{}
-		if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
-			magnetAudio = append(magnetAudio, audio...)
-		} else if len(audio) > 1 {
-			// remove portuguese audio, and append to magnetAudio
-			for _, a := range audio {
-				if a != schema.AudioPortuguese {
-					magnetAudio = append(magnetAudio, a)
-				}
-			}
-		} else {
-			magnetAudio = append(magnetAudio, audio...)
-		}
-
-		peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		title := processTitle(releaseTitle, magnetAudio)
-
-		ixt := schema.IndexedTorrent{
-			Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
-			OriginalTitle: title,
-			Audio:         magnetAudio,
-			MagnetLink:    req.MagnetLink,
-			InfoHash:      infoHash,
-			Trackers:      trackers,
-			LeechCount:    peer,
-			SeedCount:     seed,
-		}
-
-		// write to redis
-		indexedTorrents = append(indexedTorrents, ixt)
-		out, err := json.Marshal(indexedTorrents)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		err = i.redis.SetWithExpiration(ctx, manualTorrentsRedisKey, out, manualTorrentExpiration)
-		if err != nil {
-			w.WriteHeader(http.StatusInternalServerError)
-			err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-			if err != nil {
-				fmt.Println(err)
-			}
-			i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
-			return
-		}
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
-	})
-	if err != nil {
-		fmt.Println(err)
-	}
-}

api/search.go

@@ -1,55 +0,0 @@
-package handler
-
-import (
-	"encoding/json"
-	"net/http"
-	"strconv"
-
-	meilisearch "github.com/felipemarinho97/torrent-indexer/search"
-)
-
-// MeilisearchHandler handles HTTP requests for Meilisearch integration.
-type MeilisearchHandler struct {
-	Module *meilisearch.SearchIndexer
-}
-
-// NewMeilisearchHandler creates a new instance of MeilisearchHandler.
-func NewMeilisearchHandler(module *meilisearch.SearchIndexer) *MeilisearchHandler {
-	return &MeilisearchHandler{Module: module}
-}
-
-// SearchTorrentHandler handles the searching of torrent items.
-func (h *MeilisearchHandler) SearchTorrentHandler(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodGet {
-		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
-		return
-	}
-
-	query := r.URL.Query().Get("q")
-	if query == "" {
-		http.Error(w, "Query parameter 'q' is required", http.StatusBadRequest)
-		return
-	}
-
-	limitStr := r.URL.Query().Get("limit")
-	limit := 10 // Default limit
-	if limitStr != "" {
-		var err error
-		limit, err = strconv.Atoi(limitStr)
-		if err != nil || limit <= 0 {
-			http.Error(w, "Invalid limit parameter", http.StatusBadRequest)
-			return
-		}
-	}
-
-	results, err := h.Module.SearchTorrent(query, limit)
-	if err != nil {
-		http.Error(w, "Failed to search torrents", http.StatusInternalServerError)
-		return
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	if err := json.NewEncoder(w).Encode(results); err != nil {
-		http.Error(w, "Failed to encode response", http.StatusInternalServerError)
-	}
-}

api/starck_filmes.go

@@ -1,253 +0,0 @@
-package handler
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"net/url"
-	"slices"
-	"strings"
-	"time"
-
-	"github.com/PuerkitoBio/goquery"
-	"github.com/hbollon/go-edlib"
-
-	"github.com/felipemarinho97/torrent-indexer/magnet"
-	"github.com/felipemarinho97/torrent-indexer/schema"
-	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
-	"github.com/felipemarinho97/torrent-indexer/utils"
-)
-
-var starck_filmes = IndexerMeta{
-	URL:       "https://www.starckfilmes.online/",
-	SearchURL: "?s=",
-}
-
-func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Request) {
-	start := time.Now()
-	defer func() {
-		i.metrics.IndexerDuration.WithLabelValues("starck_filmes").Observe(time.Since(start).Seconds())
-		i.metrics.IndexerRequests.WithLabelValues("starck_filmes").Inc()
-	}()
-
-	ctx := r.Context()
-	// supported query params: q, page, filter_results
-	q := r.URL.Query().Get("q")
-	page := r.URL.Query().Get("page")
-
-	// URL encode query param
-	q = url.QueryEscape(q)
-	url := starck_filmes.URL
-	if q != "" {
-		url = fmt.Sprintf("%s%s%s", url, starck_filmes.SearchURL, q)
-	} else if page != "" {
-		url = fmt.Sprintf("%spage/%s", url, page)
-	}
-
-	fmt.Println("URL:>", url)
-	resp, err := i.requester.GetDocument(ctx, url)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-		if err != nil {
-			fmt.Println(err)
-		}
-		i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
-		return
-	}
-	defer resp.Close()
-
-	doc, err := goquery.NewDocumentFromReader(resp)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
-		return
-	}
-
-	var links []string
-	doc.Find(".item").Each(func(i int, s *goquery.Selection) {
-		link, _ := s.Find("div.sub-item > a").Attr("href")
-		links = append(links, link)
-	})
-
-	var itChan = make(chan []schema.IndexedTorrent)
-	var errChan = make(chan error)
-	indexedTorrents := []schema.IndexedTorrent{}
-	for _, link := range links {
-		go func(link string) {
-			torrents, err := getTorrentStarckFilmes(ctx, i, link)
-			if err != nil {
-				fmt.Println(err)
-				errChan <- err
-			}
-			itChan <- torrents
-		}(link)
-	}
-
-	for i := 0; i < len(links); i++ {
-		select {
-		case torrents := <-itChan:
-			indexedTorrents = append(indexedTorrents, torrents...)
-		case err := <-errChan:
-			fmt.Println(err)
-		}
-	}
-
-	for i, it := range indexedTorrents {
-		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
-		qLower := strings.ToLower(q)
-		splitLength := 2
-		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
-	}
-
-	// remove the ones with zero similarity
-	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
-			return it.Similarity > 0
-		})
-	}
-
-	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
-		return int((j.Similarity - i.Similarity) * 1000)
-	})
-
-	// send to search index
-	go func() {
-		_ = i.search.IndexTorrents(indexedTorrents)
-	}()
-
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
-	})
-	if err != nil {
-		fmt.Println(err)
-	}
-}
-
-func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
-	var indexedTorrents []schema.IndexedTorrent
-	doc, err := getDocument(ctx, i, link)
-	if err != nil {
-		return nil, err
-	}
-
-	post := doc.Find(".post")
-	capa := post.Find(".capa")
-	title := capa.Find(".post-description > h2").Text()
-	post_buttons := post.Find(".post-buttons")
-	magnets := post_buttons.Find("a[href^=\"magnet\"]")
-	var magnetLinks []string
-	magnets.Each(func(i int, s *goquery.Selection) {
-		magnetLink, _ := s.Attr("href")
-		magnetLinks = append(magnetLinks, magnetLink)
-	})
-
-	var audio []schema.Audio
-	var year string
-	var size []string
-	capa.Find(".post-description p").Each(func(i int, s *goquery.Selection) {
-		// pattern:
-		// Nome Original: 28 Weeks Later
-		// Lançamento: 2007
-		// Duração: 1h 40 min
-		// Gênero: Terror, Suspense, Ficção
-		// Formato: MKV
-		// Tamanho: 2.45 GB 
-		// Qualidade de Video: 10
-		// Qualidade do Audio: 10
-		// Idioma: Português | Inglês
-		// Legenda: Português, Inglês, Espanhol
-		var text strings.Builder
-		s.Find("span").Each(func (i int, span *goquery.Selection) {
-			text.WriteString(span.Text())
-			text.WriteString(" ")
-		})
-		fmt.Println(text.String())
-		audio = append(audio, findAudioFromText(text.String())...)
-		y := findYearFromText(text.String(), title)
-		if y != "" {
-			year = y
-		}
-		size = append(size, findSizesFromText(text.String())...)
-	})
-
-	// TODO: find any link from imdb
-	imdbLink := ""
-
-	size = stableUniq(size)
-
-	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
-
-	// for each magnet link, create a new indexed torrent
-	for it, magnetLink := range magnetLinks {
-		it := it
-		go func(it int, magnetLink string) {
-			magnet, err := magnet.ParseMagnetUri(magnetLink)
-			if err != nil {
-				fmt.Println(err)
-			}
-			releaseTitle := magnet.DisplayName
-			infoHash := magnet.InfoHash.String()
-			trackers := magnet.Trackers
-			magnetAudio := []schema.Audio{}
-			if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
-				magnetAudio = append(magnetAudio, audio...)
-			} else if len(audio) > 1 {
-				// remove portuguese audio, and append to magnetAudio
-				for _, a := range audio {
-					if a != schema.AudioPortuguese {
-						magnetAudio = append(magnetAudio, a)
-					}
-				}
-			} else {
-				magnetAudio = append(magnetAudio, audio...)
-			}
-
-			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
-			if err != nil {
-				fmt.Println(err)
-			}
-
-			title := processTitle(title, magnetAudio)
-
-			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
-			var mySize string
-			if len(size) == len(magnetLinks) {
-				mySize = size[it]
-			}
-
-			ixt := schema.IndexedTorrent{
-				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
-				OriginalTitle: title,
-				Details:       link,
-				Year:          year,
-				IMDB:          imdbLink,
-				Audio:         magnetAudio,
-				MagnetLink:    magnetLink,
-				Date:		   time.Now(),
-				InfoHash:      infoHash,
-				Trackers:      trackers,
-				LeechCount:    peer,
-				SeedCount:     seed,
-				Size:          mySize,
-			}
-			chanIndexedTorrent <- ixt
-		}(it, magnetLink)
-	}
-
-	for i := 0; i < len(magnetLinks); i++ {
-		it := <-chanIndexedTorrent
-		indexedTorrents = append(indexedTorrents, it)
-	}
-
-	return indexedTorrents, nil
-}

api/torrent_dos_filmes.go

@@ -1,273 +0,0 @@
-package handler
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"net/url"
-	"slices"
-	"strings"
-	"time"
-
-	"github.com/PuerkitoBio/goquery"
-	"github.com/hbollon/go-edlib"
-
-	"github.com/felipemarinho97/torrent-indexer/magnet"
-	"github.com/felipemarinho97/torrent-indexer/schema"
-	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
-	"github.com/felipemarinho97/torrent-indexer/utils"
-)
-
-var torrent_dos_filmes = IndexerMeta{
-	URL:       "https://torrentdosfilmes.se/",
-	SearchURL: "?s=",
-}
-
-func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.Request) {
-	start := time.Now()
-	defer func() {
-		i.metrics.IndexerDuration.WithLabelValues("torrent_dos_filmes").Observe(time.Since(start).Seconds())
-		i.metrics.IndexerRequests.WithLabelValues("torrent_dos_filmes").Inc()
-	}()
-
-	ctx := r.Context()
-	// supported query params: q, season, episode, page, filter_results
-	q := r.URL.Query().Get("q")
-	page := r.URL.Query().Get("page")
-
-	// URL encode query param
-	q = url.QueryEscape(q)
-	url := torrent_dos_filmes.URL
-	if q != "" {
-		url = fmt.Sprintf("%s%s%s", url, torrent_dos_filmes.SearchURL, q)
-	} else if page != "" {
-		url = fmt.Sprintf("%spage/%s", url, page)
-	}
-
-	fmt.Println("URL:>", url)
-	resp, err := i.requester.GetDocument(ctx, url)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-		if err != nil {
-			fmt.Println(err)
-		}
-		i.metrics.IndexerErrors.WithLabelValues("torrent_dos_filmes").Inc()
-		return
-	}
-	defer resp.Close()
-
-	doc, err := goquery.NewDocumentFromReader(resp)
-	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		i.metrics.IndexerErrors.WithLabelValues("torrent_dos_filmes").Inc()
-		return
-	}
-
-	var links []string
-	doc.Find(".post").Each(func(i int, s *goquery.Selection) {
-		link, _ := s.Find("div.title > a").Attr("href")
-		links = append(links, link)
-	})
-
-	var itChan = make(chan []schema.IndexedTorrent)
-	var errChan = make(chan error)
-	indexedTorrents := []schema.IndexedTorrent{}
-	for _, link := range links {
-		go func(link string) {
-			torrents, err := getTorrentsTorrentDosFilmes(ctx, i, link)
-			if err != nil {
-				fmt.Println(err)
-				errChan <- err
-			}
-			itChan <- torrents
-		}(link)
-	}
-
-	for i := 0; i < len(links); i++ {
-		select {
-		case torrents := <-itChan:
-			indexedTorrents = append(indexedTorrents, torrents...)
-		case err := <-errChan:
-			fmt.Println(err)
-		}
-	}
-
-	for i, it := range indexedTorrents {
-		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
-		qLower := strings.ToLower(q)
-		splitLength := 2
-		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
-	}
-
-	// remove the ones with zero similarity
-	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
-			return it.Similarity > 0
-		})
-	}
-
-	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
-		return int((j.Similarity - i.Similarity) * 1000)
-	})
-
-	// send to search index
-	go func() {
-		_ = i.search.IndexTorrents(indexedTorrents)
-	}()
-
-	w.Header().Set("Content-Type", "application/json")
-	err = json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
-	})
-	if err != nil {
-		fmt.Println(err)
-	}
-}
-
-func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
-	var indexedTorrents []schema.IndexedTorrent
-	doc, err := getDocument(ctx, i, link)
-	if err != nil {
-		return nil, err
-	}
-
-	article := doc.Find("article")
-	title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1)
-	textContent := article.Find("div.content")
-	date := getPublishedDateTDF(doc)
-	magnets := textContent.Find("a[href^=\"magnet\"]")
-	var magnetLinks []string
-	magnets.Each(func(i int, s *goquery.Selection) {
-		magnetLink, _ := s.Attr("href")
-		magnetLinks = append(magnetLinks, magnetLink)
-	})
-
-	var audio []schema.Audio
-	var year string
-	var size []string
-	article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
-		// pattern:
-		// Título Traduzido: Fundação
-		// Título Original: Foundation
-		// IMDb: 7,5
-		// Ano de Lançamento: 2023
-		// Gênero: Ação | Aventura | Ficção
-		// Formato: MKV
-		// Qualidade: WEB-DL
-		// Áudio: Português | Inglês
-		// Idioma: Português | Inglês
-		// Legenda: Português
-		// Tamanho: –
-		// Qualidade de Áudio: 10
-		// Qualidade de Vídeo: 10
-		// Duração: 59 Min.
-		// Servidor: Torrent
-		text := s.Text()
-
-		audio = append(audio, findAudioFromText(text)...)
-		y := findYearFromText(text, title)
-		if y != "" {
-			year = y
-		}
-		size = append(size, findSizesFromText(text)...)
-	})
-
-	// find any link from imdb
-	imdbLink := ""
-	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
-		link, _ := s.Attr("href")
-		_imdbLink, err := getIMDBLink(link)
-		if err == nil {
-			imdbLink = _imdbLink
-		}
-	})
-
-	size = stableUniq(size)
-
-	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
-
-	// for each magnet link, create a new indexed torrent
-	for it, magnetLink := range magnetLinks {
-		it := it
-		go func(it int, magnetLink string) {
-			magnet, err := magnet.ParseMagnetUri(magnetLink)
-			if err != nil {
-				fmt.Println(err)
-			}
-			releaseTitle := magnet.DisplayName
-			infoHash := magnet.InfoHash.String()
-			trackers := magnet.Trackers
-			magnetAudio := []schema.Audio{}
-			if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
-				magnetAudio = append(magnetAudio, audio...)
-			} else if len(audio) > 1 {
-				// remove portuguese audio, and append to magnetAudio
-				for _, a := range audio {
-					if a != schema.AudioPortuguese {
-						magnetAudio = append(magnetAudio, a)
-					}
-				}
-			} else {
-				magnetAudio = append(magnetAudio, audio...)
-			}
-
-			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
-			if err != nil {
-				fmt.Println(err)
-			}
-
-			title := processTitle(title, magnetAudio)
-
-			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
-			var mySize string
-			if len(size) == len(magnetLinks) {
-				mySize = size[it]
-			}
-
-			ixt := schema.IndexedTorrent{
-				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
-				OriginalTitle: title,
-				Details:       link,
-				Year:          year,
-				IMDB:          imdbLink,
-				Audio:         magnetAudio,
-				MagnetLink:    magnetLink,
-				Date:          date,
-				InfoHash:      infoHash,
-				Trackers:      trackers,
-				LeechCount:    peer,
-				SeedCount:     seed,
-				Size:          mySize,
-			}
-			chanIndexedTorrent <- ixt
-		}(it, magnetLink)
-	}
-
-	for i := 0; i < len(magnetLinks); i++ {
-		it := <-chanIndexedTorrent
-		indexedTorrents = append(indexedTorrents, it)
-	}
-
-	return indexedTorrents, nil
-}
-
-func getPublishedDateTDF(document *goquery.Document) time.Time {
-	var date time.Time
-	//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">
-	datePublished := strings.TrimSpace(document.Find("meta[property=\"article:published_time\"]").AttrOr("content", ""))
-
-	if datePublished != "" {
-		date, _ = time.Parse(time.RFC3339, datePublished)
-	}
-
-	return date
-}

cache/redis.go

@@ -9,14 +9,13 @@ import (
 	"github.com/redis/go-redis/v9"
 )
 
-const (
+var (
-	DefaultExpiration      = 24 * time.Hour * 7 // 7 days
+	DefaultExpiration      = 24 * time.Hour * 180 // 180 days
 	IndexerComandoTorrents = "indexer:comando_torrents"
 )
 
 type Redis struct {
 	client *redis.Client
-	defaultExpiration time.Duration
 }
 
 func NewRedis() *Redis {
@@ -29,20 +28,15 @@ func NewRedis() *Redis {
 			Addr:     fmt.Sprintf("%s:6379", redisHost),
 			Password: "",
 		}),
-		defaultExpiration: DefaultExpiration,
 	}
 }
 
-func (r *Redis) SetDefaultExpiration(expiration time.Duration) {
-	r.defaultExpiration = expiration
-}
-
 func (r *Redis) Get(ctx context.Context, key string) ([]byte, error) {
 	return r.client.Get(ctx, key).Bytes()
 }
 
 func (r *Redis) Set(ctx context.Context, key string, value []byte) error {
-	return r.client.Set(ctx, key, value, r.defaultExpiration).Err()
+	return r.client.Set(ctx, key, value, DefaultExpiration).Err()
 }
 
 func (r *Redis) SetWithExpiration(ctx context.Context, key string, value []byte, expiration time.Duration) error {

docker-compose.yml

@@ -1,8 +1,11 @@
-version: '3'
+version: '3.7'
 
 services:
   torrent-indexer:
-    image: felipemarinho97/torrent-indexer:latest
+    image:
+      build:
+        context: .
+        dockerfile: Dockerfile
     container_name: torrent-indexer
     restart: unless-stopped
     ports:
@@ -11,9 +14,6 @@ services:
       - indexer
     environment:
       - REDIS_HOST=redis
-      - MEILISEARCH_ADDRESS=http://meilisearch:7700
-      - MEILISEARCH_KEY=my-secret-key
-      - FLARESOLVERR_ADDRESS=http://flaresolverr:8191
   
   redis:
     image: redis:alpine
@@ -22,17 +22,5 @@ services:
     networks:
       - indexer
 
-  # This container is not necessary for the indexer to work,
-  # deploy if you want to use the search feature
-  meilisearch: 
-    image: getmeili/meilisearch:latest
-    container_name: meilisearch
-    restart: unless-stopped
-    networks:
-      - indexer
-    environment:
-      - MEILI_NO_ANALYTICS=true
-      - MEILI_MASTER_KEY=my-secret-key
-
 networks:
   indexer:

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/prometheus/client_model v0.6.0 // indirect
 	github.com/prometheus/common v0.50.0 // indirect
 	github.com/prometheus/procfs v0.13.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 )
@@ -20,6 +21,4 @@ require (
 	github.com/PuerkitoBio/goquery v1.9.1
 	github.com/hbollon/go-edlib v1.6.0
 	github.com/prometheus/client_golang v1.19.0
-	github.com/xhit/go-str2duration/v2 v2.1.0
-	golang.org/x/net v0.22.0
 )

go.sum

@@ -28,8 +28,6 @@ github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGK
 github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
 github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
 github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
-github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
-github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

main.go

@@ -1,59 +1,26 @@
 package main
 
 import (
-	"fmt"
 	"net/http"
-	"os"
 
 	handler "github.com/felipemarinho97/torrent-indexer/api"
 	"github.com/felipemarinho97/torrent-indexer/cache"
 	"github.com/felipemarinho97/torrent-indexer/monitoring"
-	"github.com/felipemarinho97/torrent-indexer/public"
-	"github.com/felipemarinho97/torrent-indexer/requester"
-	meilisearch "github.com/felipemarinho97/torrent-indexer/search"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-
-	str2duration "github.com/xhit/go-str2duration/v2"
 )
 
 func main() {
 	redis := cache.NewRedis()
-	searchIndex := meilisearch.NewSearchIndexer(os.Getenv("MEILISEARCH_ADDRESS"), os.Getenv("MEILISEARCH_KEY"), "torrents")
 	metrics := monitoring.NewMetrics()
 	metrics.Register()
-
+	indexers := handler.NewIndexers(redis, metrics)
-	flaresolverr := requester.NewFlareSolverr(os.Getenv("FLARESOLVERR_ADDRESS"), 60000)
-	req := requester.NewRequester(flaresolverr, redis)
-
-	// get shot-lived and long-lived cache expiration from env
-	shortLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("SHORT_LIVED_CACHE_EXPIRATION"))
-	if err == nil {
-		fmt.Printf("Setting short-lived cache expiration to %s\n", shortLivedCacheExpiration)
-		req.SetShortLivedCacheExpiration(shortLivedCacheExpiration)
-	}
-	longLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("LONG_LIVED_CACHE_EXPIRATION"))
-	if err == nil {
-		fmt.Printf("Setting long-lived cache expiration to %s\n", longLivedCacheExpiration)
-		redis.SetDefaultExpiration(longLivedCacheExpiration)
-	} else {
-		fmt.Println(err)
-	}
-
-	indexers := handler.NewIndexers(redis, metrics, req, searchIndex)
-	search := handler.NewMeilisearchHandler(searchIndex)
 
 	indexerMux := http.NewServeMux()
 	metricsMux := http.NewServeMux()
 
 	indexerMux.HandleFunc("/", handler.HandlerIndex)
 	indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
-	indexerMux.HandleFunc("/indexers/torrent-dos-filmes", indexers.HandlerTorrentDosFilmesIndexer)
 	indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
-	indexerMux.HandleFunc("/indexers/comandohds", indexers.HandlerComandoHDsIndexer)
-	indexerMux.HandleFunc("/indexers/starck-filmes", indexers.HandlerStarckFilmesIndexer)
-	indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer)
-	indexerMux.HandleFunc("/search", search.SearchTorrentHandler)
-	indexerMux.Handle("/ui/", http.StripPrefix("/ui/", http.FileServer(http.FS(public.UIFiles))))
 
 	metricsMux.Handle("/metrics", promhttp.Handler())
 
@@ -64,13 +31,7 @@ func main() {
 		}
 	}()
 
-	port := os.Getenv("PORT")
+	err := http.ListenAndServe(":7006", indexerMux)
-	if port == "" {
-		port = "7006"
-	}
-
-	fmt.Printf("Server listening on :%s\n", port)
-	err = http.ListenAndServe(":"+port, indexerMux)
 	if err != nil {
 		panic(err)
 	}

public/index.go

@@ -1,6 +0,0 @@
-package public
-
-import "embed"
-
-//go:embed *
-var UIFiles embed.FS

public/index.html

@@ -1,127 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-
-<head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Torrent Indexer</title>
-  <script src="https://cdn.tailwindcss.com"></script>
-  <script src="https://cdn.jsdelivr.net/npm/@heroicons/react/solid@2.0.0/dist/index.umd.js"></script>
-</head>
-
-<body class="bg-gray-900 text-white font-sans">
-  <div class="container mx-auto p-6">
-    <!-- Header -->
-    <header class="text-center mb-10">
-      <h1 class="text-4xl font-bold text-blue-400">Torrent Indexer 🇧🇷</h1>
-      <p class="text-gray-400 mt-2">Find torrents with detailed information from torrent-indexer cache</p>
-    </header>
-
-    <!-- Search Bar -->
-    <div class="flex justify-center mb-10">
-      <input id="search-query" type="text" placeholder="Enter search query"
-        class="w-full max-w-lg px-4 py-2 rounded-md border border-gray-600 bg-gray-800 text-white focus:ring focus:ring-blue-500">
-      <button id="search-btn"
-        class="ml-4 px-6 py-2 bg-blue-600 hover:bg-blue-700 rounded-md font-bold text-white">Search</button>
-    </div>
-
-    <!-- Results Section -->
-    <div id="results" class="space-y-6">
-      <!-- Dynamic content will be injected here -->
-    </div>
-  </div>
-
-  <script>
-    // Function to render a single torrent result
-    function renderTorrent(torrent) {
-      return `
-        <div class="p-6 bg-gray-800 rounded-lg shadow-md flex flex-col md:flex-row gap-6">
-          <!-- Torrent Title and Details -->
-          <div class="flex-grow">
-            <h2 class="text-2xl font-bold text-blue-400 flex items-center gap-2">
-              <span>${torrent.title}</span>
-              <span class="text-sm text-gray-400">(${torrent.year})</span>
-            </h2>
-            <p class="text-gray-500 italic mt-1">${torrent.original_title}</p>
-            <div class="mt-4 grid grid-cols-1 sm:grid-cols-2 gap-4">
-              <p><strong>Audio:</strong> ${torrent.audio.join(', ')}</p>
-              <p><strong>Size:</strong> ${torrent.size}</p>
-              <p><strong>Seeds:</strong> ${torrent.seed_count} | <strong>Leeches:</strong> ${torrent.leech_count}</p>
-              <p><strong>Info Hash:</strong> <span class="text-sm break-all text-gray-300">${torrent.info_hash}</span></p>
-            </div>
-          </div>
-
-          <!-- Actions -->
-          <div class="flex flex-col justify-between items-start md:items-end">
-            <div>
-              <a href="${torrent.imdb}" target="_blank"
-                class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium">
-                <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
-                  <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
-                    d="M8 16l-4-4m0 0l4-4m-4 4h16" />
-                </svg>
-                View on IMDB
-              </a>
-              <a href="${torrent.details}" target="_blank"
-                class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium mt-2">
-                <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
-                  <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
-                    d="M13 16h-1v-4h-.01M9 20h6a2 2 0 002-2v-5a2 2 0 00-2-2h-3.5a2 2 0 00-1.85 1.19M13 10V6a3 3 0 00-6 0v4" />
-                </svg>
-                View Details
-              </a>
-            </div>
-            <a href="${torrent.magnet_link}" target="_blank"
-              class="px-4 py-2 bg-green-600 hover:bg-green-700 text-white font-bold rounded-md flex items-center gap-2 mt-4">
-              <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
-                <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
-                  d="M9 17v-6m6 6v-6m-6 6l-2-2m8 0l2-2M5 9l7-7 7 7" />
-              </svg>
-              Download Magnet
-            </a>
-          </div>
-        </div>
-      `;
-    }
-
-    // Handle search
-    async function onSearch() {
-      const query = document.getElementById('search-query').value.trim();
-      if (!query) {
-        alert('Please enter a search query!');
-        return;
-      }
-
-      try {
-        const response = await fetch(`/search?q=${encodeURIComponent(query)}`);
-        if (!response.ok) {
-          throw new Error('Search failed');
-        }
-
-        const results = await response.json();
-        const resultsContainer = document.getElementById('results');
-        resultsContainer.innerHTML = results.map(renderTorrent).join('');
-      } catch (error) {
-        // add error element
-        document.getElementById('results').innerHTML = `
-          <div class="p-6 bg-red-800 rounded-lg shadow-md text-center">
-            <p class="text-xl font-bold text-red-400">Error fetching search results</p>
-            <p class="text-gray-400 mt-2">Please try again later.</p>
-          </div>
-        `;
-        //alert('Error fetching search results. Please try again.');
-        console.error(error);
-      }
-    }
-    
-    document.getElementById('search-btn').addEventListener('click', onSearch);
-    // on enter press
-    document.getElementById('search-query').addEventListener('keypress', (e) => {
-      if (e.key === 'Enter') {
-        onSearch();
-      }
-    });
-  </script>
-</body>
-
-</html>

requester/flaresolverr.go

@@ -1,300 +0,0 @@
-package requester
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"net/http/cookiejar"
-	"net/url"
-	"strings"
-	"sync"
-
-	"github.com/felipemarinho97/torrent-indexer/utils"
-)
-
-type FlareSolverr struct {
-	url         string
-	maxTimeout  int
-	httpClient  *http.Client
-	sessionPool chan string
-	mu          sync.Mutex
-	initiated   bool
-}
-
-var (
-	ErrListSessions = fmt.Errorf("failed to list sessions")
-)
-
-func NewFlareSolverr(url string, timeoutMilli int) *FlareSolverr {
-	poolSize := 5
-	httpClient := &http.Client{}
-	sessionPool := make(chan string, poolSize) // Pool size of 5 sessions
-
-	f := &FlareSolverr{
-		url:         url,
-		maxTimeout:  timeoutMilli,
-		httpClient:  httpClient,
-		sessionPool: sessionPool,
-	}
-
-	err := f.FillSessionPool()
-	if err == nil {
-		f.initiated = true
-	}
-
-	return f
-}
-
-func (f *FlareSolverr) FillSessionPool() error {
-	// Check if the pool is already filled
-	if len(f.sessionPool) == cap(f.sessionPool) {
-		return nil
-	}
-
-	// Pre-initialize the pool with existing sessions
-	sessions, err := f.ListSessions()
-	if err != nil {
-		// if fail to list sessions, it may not support the sessions.list command
-		// create new dumb sessions to fill the pool
-		if err == ErrListSessions {
-			for len(f.sessionPool) < cap(f.sessionPool) {
-				f.sessionPool <- "dumb-session"
-			}
-			return nil
-		}
-		fmt.Println("Failed to list existing FlareSolverr sessions:", err)
-		return err
-	} else {
-		for _, session := range sessions {
-			// Add available sessions to the pool
-			if len(f.sessionPool) < cap(f.sessionPool) {
-				f.sessionPool <- session
-			}
-		}
-		if len(f.sessionPool) > 0 {
-			fmt.Printf("Added %d FlareSolverr sessions to the pool\n", len(f.sessionPool))
-		}
-	}
-
-	// If fewer than poolSize sessions were found, create new ones to fill the pool
-	for len(f.sessionPool) < cap(f.sessionPool) {
-		f.CreateSession()
-	}
-
-	return nil
-}
-
-func (f *FlareSolverr) CreateSession() string {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-
-	body := map[string]string{"cmd": "sessions.create"}
-	jsonBody, err := json.Marshal(body)
-	if err != nil {
-		return ""
-	}
-
-	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
-	if err != nil {
-		return ""
-	}
-
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := f.httpClient.Do(req)
-	if err != nil {
-		return ""
-	}
-
-	defer resp.Body.Close()
-
-	var sessionResponse map[string]interface{}
-	err = json.NewDecoder(resp.Body).Decode(&sessionResponse)
-	if err != nil {
-		return ""
-	}
-
-	session := sessionResponse["session"].(string)
-	// Add session to the pool
-	f.sessionPool <- session
-
-	fmt.Println("Created new FlareSolverr session:", session)
-	return session
-}
-
-func (f *FlareSolverr) ListSessions() ([]string, error) {
-	body := map[string]string{"cmd": "sessions.list"}
-	jsonBody, err := json.Marshal(body)
-	if err != nil {
-		return nil, err
-	}
-
-	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := f.httpClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-
-	defer resp.Body.Close()
-
-	var sessionsResponse map[string]interface{}
-	err = json.NewDecoder(resp.Body).Decode(&sessionsResponse)
-	if err != nil {
-		return nil, err
-	}
-	if sessionsResponse["sessions"] == nil {
-		return nil, ErrListSessions
-	}
-
-	sessions := sessionsResponse["sessions"].([]interface{})
-	var sessionIDs []string
-	for _, session := range sessions {
-		sessionIDs = append(sessionIDs, session.(string))
-	}
-
-	return sessionIDs, nil
-}
-
-func (f *FlareSolverr) RetrieveSession() string {
-	// Blocking receive from the session pool.
-	session := <-f.sessionPool
-	return session
-}
-
-type Response struct {
-	Status   string `json:"status"`
-	Message  string `json:"message"`
-	Solution struct {
-		Url     string `json:"url"`
-		Status  int    `json:"status"`
-		Cookies []struct {
-			Domain   string `json:"domain"`
-			Expiry   int    `json:"expiry"`
-			HttpOnly bool   `json:"httpOnly"`
-			Name     string `json:"name"`
-			Path     string `json:"path"`
-			SameSite string `json:"sameSite"`
-			Secure   bool   `json:"secure"`
-			Value    string `json:"value"`
-		} `json:"cookies"`
-		UserAgent string            `json:"userAgent"`
-		Headers   map[string]string `json:"headers"`
-		Response  string            `json:"response"`
-	} `json:"solution"`
-}
-
-func (f *FlareSolverr) Get(_url string) (io.ReadCloser, error) {
-	// Check if the FlareSolverr instance was initiated
-	if !f.initiated {
-		return io.NopCloser(bytes.NewReader([]byte(""))), nil
-	}
-
-	// Retrieve session from the pool (blocking if no sessions available)
-	session := f.RetrieveSession()
-
-	// Ensure the session is returned to the pool after the request is done
-	defer func() {
-		f.sessionPool <- session
-	}()
-
-	body := map[string]string{
-		"cmd":        "request.get",
-		"url":        _url,
-		"maxTimeout": fmt.Sprintf("%d", f.maxTimeout),
-		"session":    session,
-	}
-	jsonBody, err := json.Marshal(body)
-	if err != nil {
-		return nil, err
-	}
-
-	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := f.httpClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-
-	// Parse the response
-	var response Response
-	err = json.NewDecoder(resp.Body).Decode(&response)
-	if err != nil {
-		return nil, err
-	}
-
-	// Check if the response was successful
-	if response.Status != "ok" {
-		return nil, fmt.Errorf("failed to get response: %s", response.Message)
-	}
-
-	// Check if "Under attack" is in the response
-	if strings.Contains(response.Solution.Response, "Under attack") {
-		return nil, fmt.Errorf("under attack")
-	}
-
-	// check if the response is valid HTML
-	if !utils.IsValidHTML(response.Solution.Response) {
-		fmt.Printf("[FlareSolverr] Invalid HTML response from %s\n", _url)
-		response.Solution.Response = ""
-	}
-
-	// If the response body is empty but cookies are present, make a new request
-	if response.Solution.Response == "" && len(response.Solution.Cookies) > 0 {
-		fmt.Printf("[FlareSolverr] Making a new request to %s with cookies\n", _url)
-		// Create a new request with cookies
-		client := &http.Client{}
-		cookieJar, err := cookiejar.New(&cookiejar.Options{})
-		if err != nil {
-			return nil, err
-		}
-		for _, cookie := range response.Solution.Cookies {
-			cookieJar.SetCookies(&url.URL{Host: cookie.Domain}, []*http.Cookie{
-				{
-					Name:   cookie.Name,
-					Value:  cookie.Value,
-					Domain: cookie.Domain,
-					Path:   cookie.Path,
-				},
-			})
-		}
-		client.Jar = cookieJar
-
-		secondReq, err := http.NewRequest("GET", _url, nil)
-		if err != nil {
-			return nil, err
-		}
-
-		// use the same user returned by the FlareSolverr
-		secondReq.Header.Set("User-Agent", response.Solution.UserAgent)
-
-		secondResp, err := client.Do(secondReq)
-		if err != nil {
-			return nil, err
-		}
-
-		respByte := new(bytes.Buffer)
-		_, err = respByte.ReadFrom(secondResp.Body)
-		if err != nil {
-			return nil, err
-		}
-
-		// Return the body of the second request
-		return io.NopCloser(bytes.NewReader(respByte.Bytes())), nil
-	}
-
-	// Return the original response body
-	return io.NopCloser(bytes.NewReader([]byte(response.Solution.Response))), nil
-}

requester/requester.go

@@ -1,97 +0,0 @@
-package requester
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"regexp"
-	"time"
-
-	"github.com/felipemarinho97/torrent-indexer/cache"
-)
-
-const (
-	cacheKey = "shortLivedCache"
-)
-
-var challangeRegex = regexp.MustCompile(`(?i)(just a moment|cf-chl-bypass|under attack)`)
-
-type Requster struct {
-	fs                        *FlareSolverr
-	c                         *cache.Redis
-	httpClient                *http.Client
-	shortLivedCacheExpiration time.Duration
-}
-
-func NewRequester(fs *FlareSolverr, c *cache.Redis) *Requster {
-	return &Requster{fs: fs, httpClient: &http.Client{}, c: c, shortLivedCacheExpiration: 30 * time.Minute}
-}
-
-func (i *Requster) SetShortLivedCacheExpiration(expiration time.Duration) {
-	i.shortLivedCacheExpiration = expiration
-}
-
-func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser, error) {
-	var body io.ReadCloser
-
-	// try request from short-lived cache
-	key := fmt.Sprintf("%s:%s", cacheKey, url)
-	bodyByte, err := i.c.Get(ctx, key)
-	if err == nil {
-		fmt.Printf("returning from short-lived cache: %s\n", url)
-		body = io.NopCloser(bytes.NewReader(bodyByte))
-		return body, nil
-	}
-
-	// try request with plain client
-	resp, err := i.httpClient.Get(url)
-	if err != nil {
-		// try request with flare solverr
-		body, err = i.fs.Get(url)
-		if err != nil {
-			return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
-		}
-	} else {
-		defer resp.Body.Close()
-		body = resp.Body
-	}
-
-	bodyByte, err = io.ReadAll(body)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read response body: %w", err)
-	}
-	if hasChallange(bodyByte) {
-		// try request with flare solverr
-		body, err = i.fs.Get(url)
-		if err != nil {
-			return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
-		}
-		bodyByte, err = io.ReadAll(body)
-		if err != nil {
-			return nil, fmt.Errorf("failed to read response body: %w", err)
-		}
-		fmt.Printf("request served from flaresolverr: %s\n", url)
-	} else {
-		fmt.Printf("request served from plain client: %s\n", url)
-	}
-
-	// save response to cache if it's not a challange and body is not empty
-	if !hasChallange(bodyByte) && len(bodyByte) > 0 {
-		err = i.c.SetWithExpiration(ctx, key, bodyByte, i.shortLivedCacheExpiration)
-		if err != nil {
-			fmt.Printf("failed to save response to cache: %v\n", err)
-		}
-		fmt.Printf("saved to cache: %s\n", url)
-	} else {
-		return nil, fmt.Errorf("response is a challange")
-	}
-
-	return io.NopCloser(bytes.NewReader(bodyByte)), nil
-}
-
-// hasChallange checks if the body contains a challange by regex matching
-func hasChallange(body []byte) bool {
-	return challangeRegex.Match(body)
-}

schema/audio.go

@@ -1,14 +1,10 @@
 package schema
 
-import "strings"
-
 type Audio string
 
 const (
 	AudioPortuguese  = "Português"
 	AudioPortuguese2 = "Portugues"
-	AudioPortuguese3 = "PT-BR"
-	AudioPortuguese4 = "Dublado"
 	AudioEnglish     = "Inglês"
 	AudioEnglish2    = "Ingles"
 	AudioSpanish     = "Espanhol"
@@ -32,18 +28,11 @@ const (
 	AudioThai        = "Tailandês"
 	AudioThai2       = "Tailandes"
 	AudioTurkish     = "Turco"
-	AudioHindi       = "Hindi"
-	AudioFarsi       = "Persa"
-	AudioMalay       = "Malaio"
-	AudioDutch       = "Holandês"
-	AudioDutch2      = "Holandes"
 )
 
 var AudioList = []Audio{
 	AudioPortuguese,
 	AudioPortuguese2,
-	AudioPortuguese3,
-	AudioPortuguese4,
 	AudioEnglish,
 	AudioEnglish2,
 	AudioSpanish,
@@ -67,36 +56,27 @@ var AudioList = []Audio{
 	AudioThai,
 	AudioThai2,
 	AudioTurkish,
-	AudioHindi,
-	AudioFarsi,
-	AudioMalay,
-	AudioDutch,
-	AudioDutch2,
 }
 
 func (a Audio) String() string {
-	return a.toTag()
+	return a.toISO639_2()
 }
 
 func GetAudioFromString(s string) *Audio {
 	for _, a := range AudioList {
-		if strings.EqualFold(string(a), s) {
+		if string(a) == s {
 			return &a
 		}
 	}
 	return nil
 }
 
-func (a Audio) toTag() string {
+func (a Audio) toISO639_2() string {
 	switch a {
 	case AudioPortuguese:
-		return "brazilian"
+		return "por"
 	case AudioPortuguese2:
-		return "brazilian"
+		return "por"
-	case AudioPortuguese3:
-		return "brazilian"
-	case AudioPortuguese4:
-		return "brazilian"
 	case AudioEnglish:
 		return "eng"
 	case AudioEnglish2:
@@ -143,16 +123,6 @@ func (a Audio) toTag() string {
 		return "tha"
 	case AudioTurkish:
 		return "tur"
-	case AudioHindi:
-		return "hin"
-	case AudioFarsi:
-		return "fas"
-	case AudioMalay:
-		return "msa"
-	case AudioDutch:
-		return "nld"
-	case AudioDutch2:
-		return "nld"
 	default:
 		return ""
 	}

schema/indexed_torrent.go

@@ -1,20 +0,0 @@
-package schema
-
-import "time"
-
-type IndexedTorrent struct {
-	Title         string    `json:"title"`
-	OriginalTitle string    `json:"original_title"`
-	Details       string    `json:"details"`
-	Year          string    `json:"year"`
-	IMDB          string    `json:"imdb"`
-	Audio         []Audio   `json:"audio"`
-	MagnetLink    string    `json:"magnet_link"`
-	Date          time.Time `json:"date"`
-	InfoHash      string    `json:"info_hash"`
-	Trackers      []string  `json:"trackers"`
-	Size          string    `json:"size"`
-	LeechCount    int       `json:"leech_count"`
-	SeedCount     int       `json:"seed_count"`
-	Similarity    float32   `json:"similarity"`
-}

scrape/info.go

@@ -52,7 +52,7 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metric
 		fmt.Println("unable to get peers from cache for infohash:", infoHash)
 	} else {
 		m.CacheMisses.WithLabelValues("peers").Inc()
-		fmt.Println("hash:", infoHash, "get from cache -> leech:", leech, "seed:", seed)
+		fmt.Println("get from cache> leech:", leech, "seed:", seed)
 		return leech, seed, nil
 	}
 
@@ -87,18 +87,16 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metric
 	var peer peers
 	for i := 0; i < len(trackers); i++ {
 		select {
-		case <-errChan:
-			// discard error
 		case peer = <-peerChan:
 			err = setPeersToCache(ctx, r, infoHash, peer.Leechers, peer.Seeders)
 			if err != nil {
 				fmt.Println(err)
-			} else {
-				fmt.Println("hash:", infoHash, "get from tracker -> leech:", peer.Leechers, "seed:", peer.Seeders)
 			}
 			return peer.Leechers, peer.Seeders, nil
+		case err := <-errChan:
+			fmt.Println(err)
 		}
 	}
 
-	return 0, 0, fmt.Errorf("unable to get peers from trackers for infohash: %s", infoHash)
+	return 0, 0, fmt.Errorf("unable to get peers from trackers")
 }

search/meilisearch.go

@@ -1,147 +0,0 @@
-package meilisearch
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/felipemarinho97/torrent-indexer/schema"
-)
-
-// SearchIndexer integrates with Meilisearch to index and search torrent items.
-type SearchIndexer struct {
-	Client    *http.Client
-	BaseURL   string
-	APIKey    string
-	IndexName string
-}
-
-// NewSearchIndexer creates a new instance of SearchIndexer.
-func NewSearchIndexer(baseURL, apiKey, indexName string) *SearchIndexer {
-	return &SearchIndexer{
-		Client:    &http.Client{Timeout: 10 * time.Second},
-		BaseURL:   baseURL,
-		APIKey:    apiKey,
-		IndexName: indexName,
-	}
-}
-
-// IndexTorrent indexes a single torrent item in Meilisearch.
-func (t *SearchIndexer) IndexTorrent(torrent schema.IndexedTorrent) error {
-	url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
-
-	torrentWithKey := struct {
-		Hash string `json:"id"`
-		schema.IndexedTorrent
-	}{
-		Hash:           torrent.InfoHash,
-		IndexedTorrent: torrent,
-	}
-
-	jsonData, err := json.Marshal(torrentWithKey)
-	if err != nil {
-		return fmt.Errorf("failed to marshal torrent data: %w", err)
-	}
-
-	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
-	if err != nil {
-		return fmt.Errorf("failed to create request: %w", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-	if t.APIKey != "" {
-		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
-	}
-
-	resp, err := t.Client.Do(req)
-	if err != nil {
-		return fmt.Errorf("failed to execute request: %w", err)
-	}
-	defer resp.Body.Close()
-
-	return nil
-}
-
-func (t *SearchIndexer) IndexTorrents(torrents []schema.IndexedTorrent) error {
-	url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
-
-	torrentsWithKey := make([]struct {
-		Hash string `json:"id"`
-		schema.IndexedTorrent
-	}, 0, len(torrents))
-	for _, torrent := range torrents {
-		torrentWithKey := struct {
-			Hash string `json:"id"`
-			schema.IndexedTorrent
-		}{
-			Hash:           torrent.InfoHash,
-			IndexedTorrent: torrent,
-		}
-		torrentsWithKey = append(torrentsWithKey, torrentWithKey)
-	}
-
-	jsonData, err := json.Marshal(torrentsWithKey)
-	if err != nil {
-		return fmt.Errorf("failed to marshal torrent data: %w", err)
-	}
-
-	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
-	if err != nil {
-		return fmt.Errorf("failed to create request: %w", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-	if t.APIKey != "" {
-		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
-	}
-
-	resp, err := t.Client.Do(req)
-	if err != nil {
-		return fmt.Errorf("failed to execute request: %w", err)
-	}
-	defer resp.Body.Close()
-
-	return nil
-}
-
-// SearchTorrent searches indexed torrents in Meilisearch based on the query.
-func (t *SearchIndexer) SearchTorrent(query string, limit int) ([]schema.IndexedTorrent, error) {
-	url := fmt.Sprintf("%s/indexes/%s/search", t.BaseURL, t.IndexName)
-	requestBody := map[string]string{
-		"q": query,
-	}
-	jsonData, err := json.Marshal(requestBody)
-	if err != nil {
-		return nil, fmt.Errorf("failed to marshal search query: %w", err)
-	}
-
-	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
-	if err != nil {
-		return nil, fmt.Errorf("failed to create request: %w", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-	if t.APIKey != "" {
-		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
-	}
-
-	resp, err := t.Client.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("failed to execute request: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		body, _ := io.ReadAll(resp.Body)
-		return nil, fmt.Errorf("search failed: %s", body)
-	}
-
-	var result struct {
-		Hits []schema.IndexedTorrent `json:"hits"`
-	}
-	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
-		return nil, fmt.Errorf("failed to parse search response: %w", err)
-	}
-
-	return result.Hits, nil
-}

utils/util.go

@@ -1,10 +1,5 @@
 package utils
 
-import (
-	"strings"
-	"golang.org/x/net/html"
-)
-
 func Filter[A any](arr []A, f func(A) bool) []A {
 	var res []A
 	res = make([]A, 0)
@@ -15,9 +10,3 @@ func Filter[A any](arr []A, f func(A) bool) []A {
 	}
 	return res
 }
-
-func IsValidHTML(input string) bool {
-	r := strings.NewReader(input)
-	_, err := html.Parse(r)
-	return err == nil
-}