hotfix: port number

* new: feat: add brand new redetorrent.com indexer

* chg: refactor: create common package

* chg: fix: comandohds formatting

* chg: fix: dual audio detection

* chg: fix: parsing issues

* chg: refactor: remove duplicated code

* chg: refactor: move test funcs to common file

README.md

@@ -11,6 +11,9 @@ Visit [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org
 - [comando-torrents](https://comando.la/)
 - [bludv](https://bludvfilmes.tv/)
 - [torrent-dos-filmes](https://torrentdosfilmes.se/)
+- [starck-filmes](https://www.starckfilmes.online/)
+- [comandohds](https://comandohds.org/)
+- [rede-torrent](https://redetorrent.com/)
 
 ## Deploy
 
@@ -23,6 +26,21 @@ docker-compose up -d
 
 The server will be available at [http://localhost:8080/](http://localhost:8080/).
 
+## Configuration
+
+You can configure the server using the following environment variables:
+  
+- `PORT`: (optional) The port that the server will listen to. Default: `7006`
+- `FLARESOLVERR_ADDRESS`: (optional) The address of the FlareSolverr instance. Default: `N/A`
+- `MEILISEARCH_ADDRESS`: (optional) The address of the MeiliSearch instance. Default: `N/A`
+- `MEILISEARCH_KEY`: (optional) The API key of the MeiliSearch instance. Default: `N/A`
+- `REDIS_HOST`: (optional) The address of the Redis instance. Default: `localhost`
+- `SHORT_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the short-lived cache in duration format. Default: `30m`
+    - This cache is used to cache homepage or search results.
+    - Example: `30m`, `1h`, `1h30m`, `1h30m30s`
+- `LONG_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the long-lived cache in duration format. Default: `7d`
+    - This cache is used to store the torrent webpages (posts). You can set it to a higher value because the torrent pages are not updated frequently.
+
 ## Integrating with Jackett
 
 You can integrate this indexer with Jackett by adding a new Torznab custom indexer. Here is an example of how to do it for the `bludv` indexer:
@@ -98,7 +116,93 @@ search:
 # json engine n/a
 ```
 
-If you have more tips on how to integrate with other torrent API clients like Prowlarr, please open a PR.
+## Integrating with Prowlarr
+
+You can integrate this indexer with Prowlarr by adding a custom definition. See [Adding a custom YML definition](https://wiki.servarr.com/prowlarr/indexers#adding-a-custom-yml-definition).
+
+```yaml
+---
+id: torrent-indexer
+name: Torrent Indexer
+description: "Indexing Brazilian Torrent websites into structured data. github.com/felipemarinho97/torrent-indexer"
+language: pt-BR
+type: public
+encoding: UTF-8
+links:
+  - http://localhost:8080/
+
+caps:
+  categories:
+    Movies: Movies
+    TV: TV
+
+  modes:
+    search: [q]
+    tv-search: [q, season]
+    movie-search: [q]
+
+settings:
+  - name: indexer
+    type: select
+    label: Indexer
+    default: bludv
+    options:
+      bludv: BLUDV
+      comando_torrents: Comando Torrents
+      torrent-dos-filmes: Torrent dos Filmes
+      comandohds: Comando HDs
+      starck-filmes: Starck Filmes
+      rede_torrent: Rede Torrent
+
+search:
+  paths:
+    - path: "/indexers/{{ .Config.indexer }}"
+      response:
+        type: json
+  inputs:
+    filter_results: "true"
+    q: "{{ .Keywords }}"
+  keywordsfilters:
+    - name: tolower
+    - name: re_replace
+      args: ["(?i)(S0)(\\d{1,2})$", "temporada $2"]
+    - name: re_replace
+      args: ["(?i)(S)(\\d{1,3})$", "temporada $2"]
+
+  rows:
+    selector: $.results
+    count:
+      selector: $.count
+
+  fields:
+    download:
+      selector: magnet_link
+    title:
+      selector: title
+    description:
+      selector: original_title
+    details:
+      selector: details
+    infohash:
+      selector: info_hash
+    date:
+      selector: date
+    size:
+      selector: size
+    seeders:
+      selector: seed_count
+    leechers:
+      selector: leech_count
+    imdb:
+      selector: imdb
+    category_is_tv_show:
+      selector: title
+      filters:
+        - name: regexp
+          args: "\\b(S\\d+(?:E\\d+)?)\\b"
+    category:
+      text: "{{ if .Result.category_is_tv_show }}TV{{ else }}Movies{{ end }}"
+```
 
 # Warning
 

api/bludv.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
-	"regexp"
 	"slices"
 	"strings"
 	"time"
@@ -21,7 +20,7 @@ import (
 )
 
 var bludv = IndexerMeta{
-	URL:       "https://bludvfilmes.tv/",
+	URL:       "https://bludv.xyz/",
 	SearchURL: "?s=",
 }
 
@@ -33,13 +32,16 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 	}()
 
 	ctx := r.Context()
-	// supported query params: q, season, episode, filter_results
+	// supported query params: q, season, episode, page, filter_results
 	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
 
 	// URL encode query param
 	q = url.QueryEscape(q)
 	url := bludv.URL
-	if q != "" {
+	if page != "" {
+		url = fmt.Sprintf("%spage/%s", url, page)
+	} else {
 		url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q)
 	}
 
@@ -75,9 +77,9 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []IndexedTorrent)
+	var itChan = make(chan []schema.IndexedTorrent)
 	var errChan = make(chan error)
-	indexedTorrents := []IndexedTorrent{}
+	indexedTorrents := []schema.IndexedTorrent{}
 	for _, link := range links {
 		go func(link string) {
 			torrents, err := getTorrentsBluDV(ctx, i, link)
@@ -107,16 +109,21 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 
 	// remove the ones with zero similarity
 	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
+		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
 			return it.Similarity > 0
 		})
 	}
 
 	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
+	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
 		return int((j.Similarity - i.Similarity) * 1000)
 	})
 
+	// send to search index
+	go func() {
+		_ = i.search.IndexTorrents(indexedTorrents)
+	}()
+
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(Response{
 		Results: indexedTorrents,
@@ -127,8 +134,8 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
+func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
-	var indexedTorrents []IndexedTorrent
+	var indexedTorrents []schema.IndexedTorrent
 	doc, err := getDocument(ctx, i, link)
 	if err != nil {
 		return nil, err
@@ -145,6 +152,30 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
 		magnetLinks = append(magnetLinks, magnetLink)
 	})
 
+	adwareLinks := textContent.Find("a[href^=\"https://www.seuvideo.xyz\"]")
+	adwareLinks.Each(func(_ int, s *goquery.Selection) {
+		href, _ := s.Attr("href")
+		// extract querysting "id" from url
+		parsedUrl, err := url.Parse(href)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		magnetLink := parsedUrl.Query().Get("id")
+		magnetLinkDecoded, err := utils.DecodeAdLink(magnetLink)
+		if err != nil {
+			fmt.Printf("failed to decode ad link \"%s\": %v\n", href, err)
+			return
+		}
+
+		// if decoded magnet link is indeed a magnet link, append it
+		if strings.HasPrefix(magnetLinkDecoded, "magnet:") {
+			magnetLinks = append(magnetLinks, magnetLinkDecoded)
+		} else {
+			fmt.Printf("WARN: link \"%s\" decoding resulted in non-magnet link: %s\n", href, magnetLinkDecoded)
+		}
+	})
+
 	var audio []schema.Audio
 	var year string
 	var size []string
@@ -179,16 +210,15 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
 	imdbLink := ""
 	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
 		link, _ := s.Attr("href")
-		re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
+		_imdbLink, err := getIMDBLink(link)
-		matches := re.FindStringSubmatch(link)
+		if err == nil {
-		if len(matches) > 0 {
+			imdbLink = _imdbLink
-			imdbLink = matches[0]
 		}
 	})
 
 	size = stableUniq(size)
 
-	var chanIndexedTorrent = make(chan IndexedTorrent)
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
 
 	// for each magnet link, create a new indexed torrent
 	for it, magnetLink := range magnetLinks {
@@ -201,19 +231,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
 			releaseTitle := magnet.DisplayName
 			infoHash := magnet.InfoHash.String()
 			trackers := magnet.Trackers
-			magnetAudio := []schema.Audio{}
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
-			if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
-				magnetAudio = append(magnetAudio, audio...)
-			} else if len(audio) > 1 {
-				// remove portuguese audio, and append to magnetAudio
-				for _, a := range audio {
-					if a != schema.AudioPortuguese {
-						magnetAudio = append(magnetAudio, a)
-					}
-				}
-			} else {
-				magnetAudio = append(magnetAudio, audio...)
-			}
 
 			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
 			if err != nil {
@@ -228,7 +246,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
 				mySize = size[it]
 			}
 
-			ixt := IndexedTorrent{
+			ixt := schema.IndexedTorrent{
 				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
 				OriginalTitle: title,
 				Details:       link,

api/comando_torrents.go

@@ -49,14 +49,17 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 	}()
 
 	ctx := r.Context()
-	// supported query params: q, season, episode
+	// supported query params: q, season, episode, page, filter_results
 	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
 
 	// URL encode query param
 	q = url.QueryEscape(q)
 	url := comando.URL
 	if q != "" {
 		url = fmt.Sprintf("%s%s%s", url, comando.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf("%spage/%s", url, page)
 	}
 
 	fmt.Println("URL:>", url)
@@ -90,9 +93,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []IndexedTorrent)
+	var itChan = make(chan []schema.IndexedTorrent)
 	var errChan = make(chan error)
-	indexedTorrents := []IndexedTorrent{}
+	indexedTorrents := []schema.IndexedTorrent{}
 	for _, link := range links {
 		go func(link string) {
 			torrents, err := getTorrents(ctx, i, link)
@@ -122,16 +125,21 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 
 	// remove the ones with zero similarity
 	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
+		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
 			return it.Similarity > 0
 		})
 	}
 
 	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
+	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
 		return int((j.Similarity - i.Similarity) * 1000)
 	})
 
+	// send to search index
+	go func() {
+		_ = i.search.IndexTorrents(indexedTorrents)
+	}()
+
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(Response{
 		Results: indexedTorrents,
@@ -142,8 +150,8 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
+func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
-	var indexedTorrents []IndexedTorrent
+	var indexedTorrents []schema.IndexedTorrent
 	doc, err := getDocument(ctx, i, link)
 	if err != nil {
 		return nil, err
@@ -155,19 +163,11 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
 	// div itemprop="datePublished"
 	datePublished := strings.TrimSpace(article.Find("div[itemprop=\"datePublished\"]").Text())
 	// pattern: 10 de setembro de 2021
-	re := regexp.MustCompile(`(\d{2}) de (\w+) de (\d{4})`)
+	date, err := parseLocalizedDate(datePublished)
-	matches := re.FindStringSubmatch(datePublished)
+	if err != nil {
-	var date time.Time
+		return nil, err
-	if len(matches) > 0 {
-		day := matches[1]
-		month := matches[2]
-		year := matches[3]
-		datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
-		date, err = time.Parse("2006-01-02", datePublished)
-		if err != nil {
-			return nil, err
-		}
 	}
+
 	magnets := textContent.Find("a[href^=\"magnet\"]")
 	var magnetLinks []string
 	magnets.Each(func(i int, s *goquery.Selection) {
@@ -209,16 +209,15 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
 	imdbLink := ""
 	article.Find("a").Each(func(i int, s *goquery.Selection) {
 		link, _ := s.Attr("href")
-		re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
+		_imdbLink, err := getIMDBLink(link)
-		matches := re.FindStringSubmatch(link)
+		if err == nil {
-		if len(matches) > 0 {
+			imdbLink = _imdbLink
-			imdbLink = matches[0]
 		}
 	})
 
 	size = stableUniq(size)
 
-	var chanIndexedTorrent = make(chan IndexedTorrent)
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
 
 	// for each magnet link, create a new indexed torrent
 	for it, magnetLink := range magnetLinks {
@@ -231,19 +230,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
 			releaseTitle := magnet.DisplayName
 			infoHash := magnet.InfoHash.String()
 			trackers := magnet.Trackers
-			magnetAudio := []schema.Audio{}
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
-			if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
-				magnetAudio = append(magnetAudio, audio...)
-			} else if len(audio) > 1 {
-				// remove portuguese audio, and append to magnetAudio
-				for _, a := range audio {
-					if a != schema.AudioPortuguese {
-						magnetAudio = append(magnetAudio, a)
-					}
-				}
-			} else {
-				magnetAudio = append(magnetAudio, audio...)
-			}
 
 			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
 			if err != nil {
@@ -258,7 +245,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
 				mySize = size[it]
 			}
 
-			ixt := IndexedTorrent{
+			ixt := schema.IndexedTorrent{
 				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
 				OriginalTitle: title,
 				Details:       link,
@@ -285,6 +272,27 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
 	return indexedTorrents, nil
 }
 
+func parseLocalizedDate(datePublished string) (time.Time, error) {
+	re := regexp.MustCompile(`(\d{1,2}) de (\w+) de (\d{4})`)
+	matches := re.FindStringSubmatch(datePublished)
+	if len(matches) > 0 {
+		day := matches[1]
+		// append 0 to single digit day
+		if len(day) == 1 {
+			day = fmt.Sprintf("0%s", day)
+		}
+		month := matches[2]
+		year := matches[3]
+		datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
+		date, err := time.Parse("2006-01-02", datePublished)
+		if err != nil {
+			return time.Time{}, err
+		}
+		return date, nil
+	}
+	return time.Time{}, nil
+}
+
 func stableUniq(s []string) []string {
 	var uniq []map[string]interface{}
 	m := make(map[string]map[string]interface{})
@@ -317,56 +325,6 @@ func stableUniq(s []string) []string {
 	return uniqValues
 }
 
-func findYearFromText(text string, title string) (year string) {
-	re := regexp.MustCompile(`Lançamento: (.*)`)
-	yearMatch := re.FindStringSubmatch(text)
-	if len(yearMatch) > 0 {
-		year = yearMatch[1]
-	}
-
-	if year == "" {
-		re = regexp.MustCompile(`\((\d{4})\)`)
-		yearMatch := re.FindStringSubmatch(title)
-		if len(yearMatch) > 0 {
-			year = yearMatch[1]
-		}
-	}
-	return year
-}
-
-func findAudioFromText(text string) []schema.Audio {
-	var audio []schema.Audio
-	re := regexp.MustCompile(`(.udio|Idioma):.?(.*)`)
-	audioMatch := re.FindStringSubmatch(text)
-	if len(audioMatch) > 0 {
-		sep := getSeparator(audioMatch[2])
-		langs_raw := strings.Split(audioMatch[2], sep)
-		for _, lang := range langs_raw {
-			lang = strings.TrimSpace(lang)
-			a := schema.GetAudioFromString(lang)
-			if a != nil {
-				audio = append(audio, *a)
-			} else {
-				fmt.Println("unknown language:", lang)
-			}
-		}
-	}
-	return audio
-}
-
-func findSizesFromText(text string) []string {
-	var sizes []string
-	// everything that ends with GB or MB, using ',' or '.' as decimal separator
-	re := regexp.MustCompile(`(\d+[\.,]?\d+) ?(GB|MB)`)
-	sizesMatch := re.FindAllStringSubmatch(text, -1)
-	if len(sizesMatch) > 0 {
-		for _, size := range sizesMatch {
-			sizes = append(sizes, size[0])
-		}
-	}
-	return sizes
-}
-
 func processTitle(title string, a []schema.Audio) string {
 	// remove ' - Donwload' from title
 	title = strings.Replace(title, " – Download", "", -1)
@@ -380,26 +338,6 @@ func processTitle(title string, a []schema.Audio) string {
 	return title
 }
 
-func appendAudioISO639_2Code(title string, a []schema.Audio) string {
-	if len(a) > 0 {
-		audio := []string{}
-		for _, lang := range a {
-			audio = append(audio, lang.String())
-		}
-		title = fmt.Sprintf("%s (%s)", title, strings.Join(audio, ", "))
-	}
-	return title
-}
-
-func getSeparator(s string) string {
-	if strings.Contains(s, "|") {
-		return "|"
-	} else if strings.Contains(s, ",") {
-		return ","
-	}
-	return " "
-}
-
 func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Document, error) {
 	// try to get from redis first
 	docCache, err := i.redis.Get(ctx, link)

api/comando_torrents_test.go

@@ -3,76 +3,45 @@ package handler
 import (
 	"reflect"
 	"testing"
-
+	"time"
-	"github.com/felipemarinho97/torrent-indexer/schema"
 )
 
-func Test_findAudioFromText(t *testing.T) {
+func Test_parseLocalizedDate(t *testing.T) {
 	type args struct {
-		text string
+		datePublished string
 	}
 	tests := []struct {
-		name string
+		name    string
-		args args
+		args    args
-		want []schema.Audio
+		want    time.Time
+		wantErr bool
 	}{
 		{
-			name: "should return audio in portuguese",
+			name: "should return date",
 			args: args{
-				text: "Áudio: Português",
+				datePublished: "12 de outubro de 2022",
-			},
-			want: []schema.Audio{
-				schema.AudioPortuguese,
 			},
+			want:    time.Date(2022, 10, 12, 0, 0, 0, 0, time.UTC),
+			wantErr: false,
 		},
 		{
-			name: "should return audio in portuguese",
+			name: "should return date single digit",
 			args: args{
-				text: "Idioma: Português",
+				datePublished: "1 de outubro de 2022",
-			},
-			want: []schema.Audio{
-				schema.AudioPortuguese,
-			},
-		},
-		{
-			name: "should return audio in portuguese",
-			args: args{
-				text: "Audio: Português",
-			},
-			want: []schema.Audio{
-				schema.AudioPortuguese,
-			},
-		},
-		{
-			name: "should return audio in portuguese",
-			args: args{
-				text: `
-»INFORMAÇÕES«
-Título Traduzido: O Cangaceiro do Futuro
-Título Original: O Cangaceiro do Futuro
-IMDb: 7,1
-Gênero:Comédia
-Lançamento: 2022
-Qualidade: WEB-DL
-Áudio: Português
-Legenda: S/L
-Formato: MKV
-Tamanho: 5.77 GB | 9.60 GB
-Duração: 30 Min./Ep.
-Qualidade de Áudio: 10
-Qualidade de Vídeo: 10
-Servidor Via: Torrent
-				`,
-			},
-			want: []schema.Audio{
-				schema.AudioPortuguese,
 			},
+			want:    time.Date(2022, 10, 1, 0, 0, 0, 0, time.UTC),
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := findAudioFromText(tt.args.text); !reflect.DeepEqual(got, tt.want) {
+			got, err := parseLocalizedDate(tt.args.datePublished)
-				t.Errorf("findAudioFromText() = %v, want %v", got, tt.want)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseDate() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseDate() = %v, want %v", got, tt.want)
 			}
 		})
 	}

api/comandohds.go

@@ -0,0 +1,255 @@
+package handler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/hbollon/go-edlib"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+var comandohds = IndexerMeta{
+	URL:       "https://comandohds.org/",
+	SearchURL: "?s=",
+}
+
+var title_re = regexp.MustCompile(`^[(Filme)|(Série)\s]+`)
+
+func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues("comandohds").Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues("comandohds").Inc()
+	}()
+
+	ctx := r.Context()
+	// supported query params: q, page, filter_results
+	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
+
+	// URL encode query param
+	q = url.QueryEscape(q)
+	url := comandohds.URL
+	if q != "" {
+		url = fmt.Sprintf("%s%s%s", url, comandohds.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf("%spage/%s", url, page)
+	}
+
+	fmt.Println("URL:>", url)
+	resp, err := i.requester.GetDocument(ctx, url)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
+		return
+	}
+	defer resp.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
+		return
+	}
+
+	var links []string
+	doc.Find(".post").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Find("div.title > a").Attr("href")
+		links = append(links, link)
+	})
+
+	var itChan = make(chan []schema.IndexedTorrent)
+	var errChan = make(chan error)
+	indexedTorrents := []schema.IndexedTorrent{}
+	for _, link := range links {
+		go func(link string) {
+			torrents, err := getTorrentsComandoHDs(ctx, i, link)
+			if err != nil {
+				fmt.Println(err)
+				errChan <- err
+			}
+			itChan <- torrents
+		}(link)
+	}
+
+	for i := 0; i < len(links); i++ {
+		select {
+		case torrents := <-itChan:
+			indexedTorrents = append(indexedTorrents, torrents...)
+		case err := <-errChan:
+			fmt.Println(err)
+		}
+	}
+
+	for i, it := range indexedTorrents {
+		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
+		qLower := strings.ToLower(q)
+		splitLength := 2
+		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
+	}
+
+	// remove the ones with zero similarity
+	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
+		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
+			return it.Similarity > 0
+		})
+	}
+
+	// sort by similarity
+	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
+		return int((j.Similarity - i.Similarity) * 1000)
+	})
+
+	// send to search index
+	go func() {
+		_ = i.search.IndexTorrents(indexedTorrents)
+	}()
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: indexedTorrents,
+		Count:   len(indexedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
+func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+	var indexedTorrents []schema.IndexedTorrent
+	doc, err := getDocument(ctx, i, link)
+	if err != nil {
+		return nil, err
+	}
+
+	article := doc.Find("article")
+	title := title_re.ReplaceAllString(article.Find(".main_title > h1").Text(), "")
+	textContent := article.Find("div.content")
+	date := getPublishedDateFromMeta(doc)
+	magnets := textContent.Find("a[href^=\"magnet\"]")
+	var magnetLinks []string
+	magnets.Each(func(i int, s *goquery.Selection) {
+		magnetLink, _ := s.Attr("href")
+		magnetLinks = append(magnetLinks, magnetLink)
+	})
+
+	var audio []schema.Audio
+	var year string
+	var size []string
+	article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
+		// pattern:
+		// »INFORMAÇÕES«
+		// Titulo Traduzido: O Guerreiro Banido
+		// Titulo Original: 天龍八部之喬峰傳
+		// <picture />: 5.7
+		// Ano de Lançamento: 2023
+		// Gênero: Ação
+		// Formato: MKV
+		// Qualidade: WEB-DL
+		// Idioma: Português | Inglês
+		// Legenda: Português
+		// Tamanho: – GB
+		// Qualidade Áudio e Vídeo: 10
+		// Duração: 130 Min
+		// Servidor: Torrent
+		text := s.Text()
+
+		audio = append(audio, findAudioFromText(text)...)
+		y := findYearFromText(text, title)
+		if y != "" {
+			year = y
+		}
+		size = append(size, findSizesFromText(text)...)
+	})
+
+	// find any link from imdb
+	imdbLink := ""
+	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Attr("href")
+		_imdbLink, err := getIMDBLink(link)
+		if err == nil {
+			imdbLink = _imdbLink
+		}
+	})
+
+	size = stableUniq(size)
+
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
+
+	// for each magnet link, create a new indexed torrent
+	for it, magnetLink := range magnetLinks {
+		it := it
+		go func(it int, magnetLink string) {
+			magnet, err := magnet.ParseMagnetUri(magnetLink)
+			if err != nil {
+				fmt.Println(err)
+			}
+			releaseTitle := strings.TrimSpace(magnet.DisplayName)
+			infoHash := magnet.InfoHash.String()
+			trackers := magnet.Trackers
+			for i, tracker := range trackers {
+				trackers[i] = strings.TrimSpace(tracker)
+			}
+
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
+
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+			if err != nil {
+				fmt.Println(err)
+			}
+
+			title := processTitle(title, magnetAudio)
+
+			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
+			var mySize string
+			if len(size) == len(magnetLinks) {
+				mySize = size[it]
+			}
+
+			ixt := schema.IndexedTorrent{
+				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				OriginalTitle: title,
+				Details:       link,
+				Year:          year,
+				IMDB:          imdbLink,
+				Audio:         magnetAudio,
+				MagnetLink:    magnetLink,
+				Date:          date,
+				InfoHash:      infoHash,
+				Trackers:      trackers,
+				LeechCount:    peer,
+				SeedCount:     seed,
+				Size:          mySize,
+			}
+			chanIndexedTorrent <- ixt
+		}(it, magnetLink)
+	}
+
+	for i := 0; i < len(magnetLinks); i++ {
+		it := <-chanIndexedTorrent
+		indexedTorrents = append(indexedTorrents, it)
+	}
+
+	return indexedTorrents, nil
+}

api/common.go

@@ -0,0 +1,187 @@
+package handler
+
+import (
+	"fmt"
+	"regexp"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+)
+
+func getPublishedDateFromMeta(document *goquery.Document) time.Time {
+	var date time.Time
+	//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">
+	datePublished := strings.TrimSpace(document.Find("meta[property=\"article:published_time\"]").AttrOr("content", ""))
+
+	if datePublished != "" {
+		date, _ = time.Parse(time.RFC3339, datePublished)
+	}
+
+	return date
+}
+
+type datePattern struct {
+	regex  *regexp.Regexp
+	layout string
+}
+
+var datePatterns = []datePattern{
+	{regexp.MustCompile(`\d{4}-\d{2}-\d{2}`), "2006-01-02"},
+	{regexp.MustCompile(`\d{2}-\d{2}-\d{4}`), "02-01-2006"},
+	{regexp.MustCompile(`\d{2}/\d{2}/\d{4}`), "02/01/2006"},
+}
+
+// getPublishedDateFromRawString extracts the date from a raw string using predefined patterns.
+func getPublishedDateFromRawString(dateStr string) time.Time {
+	for _, p := range datePatterns {
+		match := p.regex.FindString(dateStr)
+
+		if match != "" {
+			date, err := time.Parse(p.layout, match)
+			if err == nil {
+				return date.UTC()
+			}
+		}
+	}
+
+	return time.Time{}
+}
+
+// getSeparator returns the separator used in the string.
+// It checks for common separators like "|", ",", "/", and " e "
+func getSeparator(s string) string {
+	if strings.Contains(s, "|") {
+		return "|"
+	} else if strings.Contains(s, ",") {
+		return ","
+	} else if strings.Contains(s, "/") {
+		return "/"
+	} else if strings.Contains(s, " e ") {
+		return " e "
+	}
+	return " "
+}
+
+// findAudioFromText extracts audio languages from a given text.
+// It looks for patterns like "Áudio: Português, Inglês" or "Idioma: Português, Inglês"
+func findAudioFromText(text string) []schema.Audio {
+	var audio []schema.Audio
+	re := regexp.MustCompile(`(.udio|Idioma):.?(.*)`)
+	audioMatch := re.FindStringSubmatch(text)
+	if len(audioMatch) > 0 {
+		sep := getSeparator(audioMatch[2])
+		langs_raw := strings.Split(audioMatch[2], sep)
+		for _, lang := range langs_raw {
+			lang = strings.TrimSpace(lang)
+			a := schema.GetAudioFromString(lang)
+			if a != nil {
+				audio = append(audio, *a)
+			} else {
+				fmt.Println("unknown language:", lang)
+			}
+		}
+	}
+	return audio
+}
+
+// findYearFromText extracts the year from a given text.
+// It looks for patterns like "Lançamento: 2001" in the title.
+func findYearFromText(text string, title string) (year string) {
+	re := regexp.MustCompile(`Lançamento: (.*)`)
+	yearMatch := re.FindStringSubmatch(text)
+	if len(yearMatch) > 0 {
+		year = yearMatch[1]
+	}
+
+	if year == "" {
+		re = regexp.MustCompile(`\((\d{4})\)`)
+		yearMatch := re.FindStringSubmatch(title)
+		if len(yearMatch) > 0 {
+			year = yearMatch[1]
+		}
+	}
+	return strings.TrimSpace(year)
+}
+
+// findSizesFromText extracts sizes from a given text.
+// It looks for patterns like "Tamanho: 1.26 GB" or "Tamanho: 700 MB".
+func findSizesFromText(text string) []string {
+	var sizes []string
+	// everything that ends with GB or MB, using ',' or '.' as decimal separator
+	re := regexp.MustCompile(`(\d+[\.,]?\d+) ?(GB|MB)`)
+	sizesMatch := re.FindAllStringSubmatch(text, -1)
+	if len(sizesMatch) > 0 {
+		for _, size := range sizesMatch {
+			sizes = append(sizes, size[0])
+		}
+	}
+	return sizes
+}
+
+// getIMDBLink extracts the IMDB link from a given link.
+// It looks for patterns like "https://www.imdb.com/title/tt1234567/".
+// Returns an error if no valid IMDB link is found.
+func getIMDBLink(link string) (string, error) {
+	var imdbLink string
+	re := regexp.MustCompile(`https://www.imdb.com(/[a-z]{2})?/title/(tt\d+)/?`)
+
+	matches := re.FindStringSubmatch(link)
+	if len(matches) > 0 {
+		imdbLink = matches[0]
+	} else {
+		return "", fmt.Errorf("no imdb link found")
+	}
+	return imdbLink, nil
+}
+
+// appendAudioISO639_2Code appends the audio languages to the title in ISO 639-2 code format.
+// It formats the title to include the audio languages in parentheses.
+// Example: "Movie Title (eng, por)"
+func appendAudioISO639_2Code(title string, a []schema.Audio) string {
+	if len(a) > 0 {
+		audio := []string{}
+		for _, lang := range a {
+			audio = append(audio, lang.String())
+		}
+		title = fmt.Sprintf("%s (%s)", title, strings.Join(audio, ", "))
+	}
+	return title
+}
+
+// getAudioFromTitle extracts audio languages from the release title.
+// It checks for common patterns like "nacional", "dual", or "dublado"
+func getAudioFromTitle(releaseTitle string, audioFromContent []schema.Audio) []schema.Audio {
+	magnetAudio := []schema.Audio{}
+	isNacional := strings.Contains(strings.ToLower(releaseTitle), "nacional")
+	if isNacional {
+		magnetAudio = append(magnetAudio, schema.AudioPortuguese)
+	}
+
+	if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
+		magnetAudio = append(magnetAudio, audioFromContent...)
+		// if Portuguese audio is not in the audio slice, append it
+		if !slices.Contains(magnetAudio, schema.AudioPortuguese) {
+			magnetAudio = append(magnetAudio, schema.AudioPortuguese)
+		}
+	} else if len(audioFromContent) > 1 {
+		// remove portuguese audio, and append to magnetAudio
+		for _, a := range audioFromContent {
+			if a != schema.AudioPortuguese {
+				magnetAudio = append(magnetAudio, a)
+			}
+		}
+	} else {
+		magnetAudio = append(magnetAudio, audioFromContent...)
+	}
+
+	// order and uniq the audio slice
+	slices.SortFunc(magnetAudio, func(a, b schema.Audio) int {
+		return strings.Compare(a.String(), b.String())
+	})
+	magnetAudio = slices.Compact(magnetAudio)
+
+	return magnetAudio
+}

api/common_test.go

@@ -0,0 +1,241 @@
+package handler
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/felipemarinho97/torrent-indexer/schema"
+)
+
+func Test_getPublishedDateFromRawString(t *testing.T) {
+	type args struct {
+		dateStr string
+	}
+	tests := []struct {
+		name string
+		args args
+		want time.Time
+	}{
+		{
+			name: "should parse date in format 2025-01-01",
+			args: args{
+				dateStr: "2025-01-01",
+			},
+			want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "should parse date in format 01-01-2025",
+			args: args{
+				dateStr: "01-01-2025",
+			},
+			want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "should parse date in format 01/01/2025",
+			args: args{
+				dateStr: "01/01/2025",
+			},
+			want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "should parse date from starck-filmes link",
+			args: args{
+				dateStr: "https://www.starckfilmes.online/catalog/jogos-de-seducao-2025-18-07-2025/",
+			},
+			want: time.Date(2025, 7, 18, 0, 0, 0, 0, time.UTC),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getPublishedDateFromRawString(tt.args.dateStr); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("getPublishedDateFromRawString() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_findAudioFromText(t *testing.T) {
+	type args struct {
+		text string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []schema.Audio
+	}{
+		{
+			name: "should return audio in portuguese",
+			args: args{
+				text: "Áudio: Português",
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+			},
+		},
+		{
+			name: "should return audio in portuguese",
+			args: args{
+				text: "Idioma: Português",
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+			},
+		},
+		{
+			name: "should return audio in portuguese",
+			args: args{
+				text: "Audio: Português",
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+			},
+		},
+		{
+			name: "should return audio in portuguese - comando_torrents",
+			args: args{
+				text: `
+»INFORMAÇÕES«
+Título Traduzido: O Cangaceiro do Futuro
+Título Original: O Cangaceiro do Futuro
+IMDb: 7,1
+Gênero:Comédia
+Lançamento: 2022
+Qualidade: WEB-DL
+Áudio: Português
+Legenda: S/L
+Formato: MKV
+Tamanho: 5.77 GB | 9.60 GB
+Duração: 30 Min./Ep.
+Qualidade de Áudio: 10
+Qualidade de Vídeo: 10
+Servidor Via: Torrent
+				`,
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+			},
+		},
+		{
+			name: "should return audio in portuguese - rede torrent",
+			args: args{
+				text: `
+Filme Bicho de Sete Cabeças Torrent
+Título Original: Bicho de Sete Cabeças
+Lançamento: 2001
+Gêneros: Drama / Nacional
+Idioma: Português
+Qualidade: 720p / BluRay
+Duração: 1h 14 Minutos
+Formato: Mp4
+Vídeo: 10 e Áudio: 10
+Legendas: Português
+Nota do Imdb: 7.7
+Tamanho: 1.26 GB
+				`,
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+			},
+		},
+		{
+			name: "should return audio in portuguese - rede torrent 2",
+			args: args{
+				text: `
+Filme Branca de Neve e o Caçador Torrent / Assistir Online
+Título Original: Snow White and the Huntsman
+Lançamento: 2012
+Gêneros: Ação / Aventura / Fantasia
+Idioma: Português / Inglês
+Duração: 126 Minutos
+Formato: Mkv / Mp4
+Vídeo: 10 e Áudio: 10
+Legendas: Sim
+Tamanho: 2.69 GB / 1.95 GB / 1.0 GB
+				`,
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+				schema.AudioEnglish,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := findAudioFromText(tt.args.text); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("findAudioFromText() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_getIMDBLink(t *testing.T) {
+	type args struct {
+		link string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "should return imdb link",
+			args: args{
+				link: "https://www.imdb.com/title/tt1234567",
+			},
+			want:    "https://www.imdb.com/title/tt1234567",
+			wantErr: false,
+		},
+		{
+			name: "should return imdb link when end with /",
+			args: args{
+				link: "https://www.imdb.com/title/tt1234567/",
+			},
+			want:    "https://www.imdb.com/title/tt1234567/",
+			wantErr: false,
+		},
+		{
+			name: "should return imdb link when end with /",
+			args: args{
+				link: "https://www.imdb.com/title/tt1234567/",
+			},
+			want:    "https://www.imdb.com/title/tt1234567/",
+			wantErr: false,
+		},
+		{
+			name: "should return imdb link when it has a language",
+			args: args{
+				link: "https://www.imdb.com/pt/title/tt18722864/",
+			},
+			want: "https://www.imdb.com/pt/title/tt18722864/",
+		},
+		{
+			name: "should return imdb link when it has a language",
+			args: args{
+				link: "https://www.imdb.com/pt/title/tt34608980/",
+			},
+			want: "https://www.imdb.com/pt/title/tt34608980/",
+		},
+		{
+			name: "should return error when link is invalid",
+			args: args{
+				link: "https://www.google.com",
+			},
+			want:    "",
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := getIMDBLink(tt.args.link)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("getIMDBLink() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("getIMDBLink() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

api/index.go

@@ -9,46 +9,34 @@ import (
 	"github.com/felipemarinho97/torrent-indexer/monitoring"
 	"github.com/felipemarinho97/torrent-indexer/requester"
 	"github.com/felipemarinho97/torrent-indexer/schema"
+	meilisearch "github.com/felipemarinho97/torrent-indexer/search"
 )
 
 type Indexer struct {
 	redis     *cache.Redis
 	metrics   *monitoring.Metrics
 	requester *requester.Requster
+	search    *meilisearch.SearchIndexer
 }
 
 type IndexerMeta struct {
 	URL       string
 	SearchURL string
+	// pattern for pagination, e.g. "page/%s"
+	PagePattern string
 }
 
 type Response struct {
-	Results []IndexedTorrent `json:"results"`
+	Results []schema.IndexedTorrent `json:"results"`
-	Count   int              `json:"count"`
+	Count   int                     `json:"count"`
 }
 
-type IndexedTorrent struct {
+func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster, si *meilisearch.SearchIndexer) *Indexer {
-	Title         string         `json:"title"`
-	OriginalTitle string         `json:"original_title"`
-	Details       string         `json:"details"`
-	Year          string         `json:"year"`
-	IMDB          string         `json:"imdb"`
-	Audio         []schema.Audio `json:"audio"`
-	MagnetLink    string         `json:"magnet_link"`
-	Date          time.Time      `json:"date"`
-	InfoHash      string         `json:"info_hash"`
-	Trackers      []string       `json:"trackers"`
-	Size          string         `json:"size"`
-	LeechCount    int            `json:"leech_count"`
-	SeedCount     int            `json:"seed_count"`
-	Similarity    float32        `json:"similarity"`
-}
-
-func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster) *Indexer {
 	return &Indexer{
 		redis:     redis,
 		metrics:   metrics,
 		requester: req,
+		search:    si,
 	}
 }
 
@@ -65,6 +53,7 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
 					"description": "Indexer for comando torrents",
 					"query_params": map[string]string{
 						"q":              "search query",
+						"page":           "page number",
 						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
 					},
 				},
@@ -75,12 +64,14 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
 					"description": "Indexer for bludv",
 					"query_params": map[string]string{
 						"q":              "search query",
+						"page":           "page number",
 						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
 					}},
 			},
 			"/indexers/torrent-dos-filmes": []map[string]interface{}{
 				{
 					"method":      "GET",
+					"page":        "page number",
 					"description": "Indexer for Torrent dos Filmes",
 					"query_params": map[string]string{
 						"q":              "search query",
@@ -88,6 +79,39 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
 					},
 				},
 			},
+			"/indexers/comandohds": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"page":        "page number",
+					"description": "Indexer for Comando HDs",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
+				},
+			},
+			"/indexers/starck-filmes": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"page":        "page number",
+					"description": "Indexer for Starck Filmes",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
+				},
+			},
+			"/indexers/rede_torrent": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"description": "Indexer for rede torrent",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"page":           "page number",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
+				},
+			},
 			"/indexers/manual": []map[string]interface{}{
 				{
 					"method":      "POST",
@@ -100,6 +124,21 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
 					"description": "Get all manual torrents",
 				},
 			},
+			"/search": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"description": "Search for cached torrents across all indexers",
+					"query_params": map[string]string{
+						"q": "search query",
+					},
+				},
+			},
+			"/ui/": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"description": "Show the unified search UI (only work if Meilisearch is enabled)",
+				},
+			},
 		},
 	})
 	if err != nil {

api/manual.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/felipemarinho97/torrent-indexer/magnet"
@@ -26,7 +25,7 @@ type ManualIndexerRequest struct {
 func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	var req ManualIndexerRequest
-	indexedTorrents := []IndexedTorrent{}
+	indexedTorrents := []schema.IndexedTorrent{}
 
 	// fetch from redis
 	out, err := i.redis.Get(ctx, manualTorrentsRedisKey)
@@ -76,19 +75,7 @@ func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
 		releaseTitle := magnet.DisplayName
 		infoHash := magnet.InfoHash.String()
 		trackers := magnet.Trackers
-		magnetAudio := []schema.Audio{}
+		magnetAudio := getAudioFromTitle(releaseTitle, audio)
-		if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
-			magnetAudio = append(magnetAudio, audio...)
-		} else if len(audio) > 1 {
-			// remove portuguese audio, and append to magnetAudio
-			for _, a := range audio {
-				if a != schema.AudioPortuguese {
-					magnetAudio = append(magnetAudio, a)
-				}
-			}
-		} else {
-			magnetAudio = append(magnetAudio, audio...)
-		}
 
 		peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
 		if err != nil {
@@ -97,7 +84,7 @@ func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
 
 		title := processTitle(releaseTitle, magnetAudio)
 
-		ixt := IndexedTorrent{
+		ixt := schema.IndexedTorrent{
 			Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
 			OriginalTitle: title,
 			Audio:         magnetAudio,

api/rede_torrent.go

@@ -0,0 +1,280 @@
+package handler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/hbollon/go-edlib"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+var rede_torrent = IndexerMeta{
+	URL:         "https://redetorrent.com/",
+	SearchURL:   "index.php?s=",
+	PagePattern: "%s",
+}
+
+func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues("rede_torrent").Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues("rede_torrent").Inc()
+	}()
+
+	ctx := r.Context()
+	// supported query params: q, season, episode, page, filter_results
+	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
+
+	// URL encode query param
+	q = url.QueryEscape(q)
+	url := rede_torrent.URL
+	if q != "" {
+		url = fmt.Sprintf("%s%s%s", url, rede_torrent.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, rede_torrent.PagePattern), page)
+	}
+
+	fmt.Println("URL:>", url)
+	resp, err := i.requester.GetDocument(ctx, url)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues("rede_torrent").Inc()
+		return
+	}
+	defer resp.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		i.metrics.IndexerErrors.WithLabelValues("rede_torrent").Inc()
+		return
+	}
+
+	var links []string
+	doc.Find(".capa_lista").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Find("a").Attr("href")
+		links = append(links, link)
+	})
+
+	var itChan = make(chan []schema.IndexedTorrent)
+	var errChan = make(chan error)
+	indexedTorrents := []schema.IndexedTorrent{}
+	for _, link := range links {
+		go func(link string) {
+			torrents, err := getTorrentsRedeTorrent(ctx, i, link)
+			if err != nil {
+				fmt.Println(err)
+				errChan <- err
+			}
+			itChan <- torrents
+		}(link)
+	}
+
+	for i := 0; i < len(links); i++ {
+		select {
+		case torrents := <-itChan:
+			indexedTorrents = append(indexedTorrents, torrents...)
+		case err := <-errChan:
+			fmt.Println(err)
+		}
+	}
+
+	for i, it := range indexedTorrents {
+		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
+		qLower := strings.ToLower(q)
+		splitLength := 2
+		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
+	}
+
+	// remove the ones with zero similarity
+	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
+		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
+			return it.Similarity > 0
+		})
+	}
+
+	// sort by similarity
+	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
+		return int((j.Similarity - i.Similarity) * 1000)
+	})
+
+	// send to search index
+	go func() {
+		_ = i.search.IndexTorrents(indexedTorrents)
+	}()
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: indexedTorrents,
+		Count:   len(indexedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
+func getTorrentsRedeTorrent(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+	var indexedTorrents []schema.IndexedTorrent
+	doc, err := getDocument(ctx, i, link)
+	if err != nil {
+		return nil, err
+	}
+
+	article := doc.Find(".conteudo")
+	// title pattern: "Something - optional balbla (dddd) some shit" - extract "Something" and "dddd"
+	titleRe := regexp.MustCompile(`^(.*?)(?: - (.*?))? \((\d{4})\)`)
+	titleP := titleRe.FindStringSubmatch(article.Find("h1").Text())
+	if len(titleP) < 3 {
+		return nil, fmt.Errorf("could not extract title from %s", link)
+	}
+	title := strings.TrimSpace(titleP[1])
+	year := strings.TrimSpace(titleP[3])
+
+	textContent := article.Find(".apenas_itemprop")
+	date := getPublishedDateFromMeta(doc)
+	magnets := textContent.Find("a[href^=\"magnet\"]")
+	var magnetLinks []string
+	magnets.Each(func(i int, s *goquery.Selection) {
+		magnetLink, _ := s.Attr("href")
+		magnetLinks = append(magnetLinks, magnetLink)
+	})
+
+	var audio []schema.Audio
+	var size []string
+	article.Find("div#informacoes > p").Each(func(i int, s *goquery.Selection) {
+		// pattern:
+		// Filme Bicho de Sete Cabeças Torrent
+		// Título Original: Bicho de Sete Cabeças
+		// Lançamento: 2001
+		// Gêneros: Drama / Nacional
+		// Idioma: Português
+		// Qualidade: 720p / BluRay
+		// Duração: 1h 14 Minutos
+		// Formato: Mp4
+		// Vídeo: 10 e Áudio: 10
+		// Legendas: Português
+		// Nota do Imdb: 7.7
+		// Tamanho: 1.26 GB
+
+		// we need to manualy parse because the text is not well formatted
+		htmlContent, err := s.Html()
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+
+		// remove any \n and \t characters
+		htmlContent = strings.ReplaceAll(htmlContent, "\n", "")
+		htmlContent = strings.ReplaceAll(htmlContent, "\t", "")
+
+		// split by <br> tags and render each line
+		brRe := regexp.MustCompile(`<br\s*\/?>`)
+		htmlContent = brRe.ReplaceAllString(htmlContent, "<br>")
+		lines := strings.Split(htmlContent, "<br>")
+
+		var text strings.Builder
+		for _, line := range lines {
+			// remove any HTML tags
+			re := regexp.MustCompile(`<[^>]*>`)
+			line = re.ReplaceAllString(line, "")
+
+			line = strings.TrimSpace(line)
+			text.WriteString(line + "\n")
+		}
+
+		audio = append(audio, findAudioFromText(text.String())...)
+		y := findYearFromText(text.String(), title)
+		if y != "" {
+			year = y
+		}
+		size = append(size, findSizesFromText(text.String())...)
+	})
+
+	// find any link from imdb
+	imdbLink := ""
+	article.Find("a").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Attr("href")
+		_imdbLink, err := getIMDBLink(link)
+		if err == nil {
+			imdbLink = _imdbLink
+		}
+	})
+
+	size = stableUniq(size)
+
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
+
+	// for each magnet link, create a new indexed torrent
+	for it, magnetLink := range magnetLinks {
+		it := it
+		go func(it int, magnetLink string) {
+			magnet, err := magnet.ParseMagnetUri(magnetLink)
+			if err != nil {
+				fmt.Println(err)
+			}
+			releaseTitle := magnet.DisplayName
+			infoHash := magnet.InfoHash.String()
+			trackers := magnet.Trackers
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
+
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+			if err != nil {
+				fmt.Println(err)
+			}
+
+			title := processTitle(title, magnetAudio)
+
+			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
+			var mySize string
+			if len(size) == len(magnetLinks) {
+				mySize = size[it]
+			}
+
+			ixt := schema.IndexedTorrent{
+				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				OriginalTitle: title,
+				Details:       link,
+				Year:          year,
+				IMDB:          imdbLink,
+				Audio:         magnetAudio,
+				MagnetLink:    magnetLink,
+				Date:          date,
+				InfoHash:      infoHash,
+				Trackers:      trackers,
+				LeechCount:    peer,
+				SeedCount:     seed,
+				Size:          mySize,
+			}
+			chanIndexedTorrent <- ixt
+		}(it, magnetLink)
+	}
+
+	for i := 0; i < len(magnetLinks); i++ {
+		it := <-chanIndexedTorrent
+		indexedTorrents = append(indexedTorrents, it)
+	}
+
+	return indexedTorrents, nil
+}

api/search.go

@@ -0,0 +1,158 @@
+package handler
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+	"time"
+
+	meilisearch "github.com/felipemarinho97/torrent-indexer/search"
+)
+
+// MeilisearchHandler handles HTTP requests for Meilisearch integration.
+type MeilisearchHandler struct {
+	Module *meilisearch.SearchIndexer
+}
+
+// HealthResponse represents the health check response
+type HealthResponse struct {
+	Status    string                 `json:"status"`
+	Service   string                 `json:"service"`
+	Details   map[string]interface{} `json:"details,omitempty"`
+	Timestamp string                 `json:"timestamp"`
+}
+
+// StatsResponse represents the stats endpoint response
+type StatsResponse struct {
+	Status            string           `json:"status"`
+	NumberOfDocuments int64            `json:"numberOfDocuments"`
+	IsIndexing        bool             `json:"isIndexing"`
+	FieldDistribution map[string]int64 `json:"fieldDistribution"`
+	Service           string           `json:"service"`
+}
+
+// NewMeilisearchHandler creates a new instance of MeilisearchHandler.
+func NewMeilisearchHandler(module *meilisearch.SearchIndexer) *MeilisearchHandler {
+	return &MeilisearchHandler{Module: module}
+}
+
+// SearchTorrentHandler handles the searching of torrent items.
+func (h *MeilisearchHandler) SearchTorrentHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	query := r.URL.Query().Get("q")
+	if query == "" {
+		http.Error(w, "Query parameter 'q' is required", http.StatusBadRequest)
+		return
+	}
+
+	limitStr := r.URL.Query().Get("limit")
+	limit := 10 // Default limit
+	if limitStr != "" {
+		var err error
+		limit, err = strconv.Atoi(limitStr)
+		if err != nil || limit <= 0 {
+			http.Error(w, "Invalid limit parameter", http.StatusBadRequest)
+			return
+		}
+	}
+
+	results, err := h.Module.SearchTorrent(query, limit)
+	if err != nil {
+		http.Error(w, "Failed to search torrents", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(results); err != nil {
+		http.Error(w, "Failed to encode response", http.StatusInternalServerError)
+	}
+}
+
+// HealthHandler provides a health check endpoint for Meilisearch.
+func (h *MeilisearchHandler) HealthHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	// Check if Meilisearch is healthy
+	isHealthy := h.Module.IsHealthy()
+
+	response := HealthResponse{
+		Service:   "meilisearch",
+		Timestamp: getCurrentTimestamp(),
+	}
+
+	if isHealthy {
+		// Try to get additional stats for more detailed health info
+		stats, err := h.Module.GetStats()
+		if err == nil {
+			response.Status = "healthy"
+			response.Details = map[string]interface{}{
+				"documents": stats.NumberOfDocuments,
+				"indexing":  stats.IsIndexing,
+			}
+			w.WriteHeader(http.StatusOK)
+		} else {
+			// Service is up but can't get stats
+			response.Status = "degraded"
+			response.Details = map[string]interface{}{
+				"error": "Could not retrieve stats",
+			}
+			w.WriteHeader(http.StatusOK)
+		}
+	} else {
+		// Service is down
+		response.Status = "unhealthy"
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}
+
+	if err := json.NewEncoder(w).Encode(response); err != nil {
+		http.Error(w, "Failed to encode response", http.StatusInternalServerError)
+	}
+}
+
+// StatsHandler provides detailed statistics about the Meilisearch index.
+func (h *MeilisearchHandler) StatsHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	// Get detailed stats from Meilisearch
+	stats, err := h.Module.GetStats()
+	if err != nil {
+		// Check if it's a connectivity issue
+		if !h.Module.IsHealthy() {
+			http.Error(w, "Meilisearch service is unavailable", http.StatusServiceUnavailable)
+			return
+		}
+		http.Error(w, "Failed to retrieve statistics", http.StatusInternalServerError)
+		return
+	}
+
+	response := StatsResponse{
+		Status:            "healthy",
+		Service:           "meilisearch",
+		NumberOfDocuments: stats.NumberOfDocuments,
+		IsIndexing:        stats.IsIndexing,
+		FieldDistribution: stats.FieldDistribution,
+	}
+
+	if err := json.NewEncoder(w).Encode(response); err != nil {
+		http.Error(w, "Failed to encode response", http.StatusInternalServerError)
+	}
+}
+
+// getCurrentTimestamp returns the current timestamp in RFC3339 format
+func getCurrentTimestamp() string {
+	return time.Now().Format(time.RFC3339)
+}

api/starck_filmes.go

@@ -0,0 +1,255 @@
+package handler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/hbollon/go-edlib"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+var starck_filmes = IndexerMeta{
+	URL:       "https://www.starckfilmes.online/",
+	SearchURL: "?s=",
+}
+
+func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues("starck_filmes").Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues("starck_filmes").Inc()
+	}()
+
+	ctx := r.Context()
+	// supported query params: q, page, filter_results
+	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
+
+	// URL encode query param
+	q = url.QueryEscape(q)
+	url := starck_filmes.URL
+	if q != "" {
+		url = fmt.Sprintf("%s%s%s", url, starck_filmes.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf("%spage/%s", url, page)
+	}
+
+	fmt.Println("URL:>", url)
+	resp, err := i.requester.GetDocument(ctx, url)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
+		return
+	}
+	defer resp.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
+		return
+	}
+
+	var links []string
+	doc.Find(".item").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Find("div.sub-item > a").Attr("href")
+		links = append(links, link)
+	})
+
+	var itChan = make(chan []schema.IndexedTorrent)
+	var errChan = make(chan error)
+	indexedTorrents := []schema.IndexedTorrent{}
+	for _, link := range links {
+		go func(link string) {
+			torrents, err := getTorrentStarckFilmes(ctx, i, link)
+			if err != nil {
+				fmt.Println(err)
+				errChan <- err
+			}
+			itChan <- torrents
+		}(link)
+	}
+
+	for i := 0; i < len(links); i++ {
+		select {
+		case torrents := <-itChan:
+			indexedTorrents = append(indexedTorrents, torrents...)
+		case err := <-errChan:
+			fmt.Println(err)
+		}
+	}
+
+	for i, it := range indexedTorrents {
+		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
+		qLower := strings.ToLower(q)
+		splitLength := 2
+		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
+	}
+
+	// remove the ones with zero similarity
+	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
+		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
+			return it.Similarity > 0
+		})
+	}
+
+	// sort by similarity
+	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
+		return int((j.Similarity - i.Similarity) * 1000)
+	})
+
+	// send to search index
+	go func() {
+		_ = i.search.IndexTorrents(indexedTorrents)
+	}()
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: indexedTorrents,
+		Count:   len(indexedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
+func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+	var indexedTorrents []schema.IndexedTorrent
+	doc, err := getDocument(ctx, i, link)
+	if err != nil {
+		return nil, err
+	}
+
+	date := getPublishedDateFromRawString(link)
+
+	post := doc.Find(".post")
+	capa := post.Find(".capa")
+	title := capa.Find(".post-description > h2").Text()
+	post_buttons := post.Find(".post-buttons")
+	magnets := post_buttons.Find("a[href^=\"magnet\"]")
+	var magnetLinks []string
+	magnets.Each(func(i int, s *goquery.Selection) {
+		magnetLink, _ := s.Attr("href")
+		magnetLinks = append(magnetLinks, magnetLink)
+	})
+
+	var audio []schema.Audio
+	var year string
+	var size []string
+	capa.Find(".post-description p").Each(func(i int, s *goquery.Selection) {
+		// pattern:
+		// Nome Original: 28 Weeks Later
+		// Lançamento: 2007
+		// Duração: 1h 40 min
+		// Gênero: Terror, Suspense, Ficção
+		// Formato: MKV
+		// Tamanho: 2.45 GB
+		// Qualidade de Video: 10
+		// Qualidade do Audio: 10
+		// Idioma: Português | Inglês
+		// Legenda: Português, Inglês, Espanhol
+		var text strings.Builder
+		s.Find("span").Each(func(i int, span *goquery.Selection) {
+			text.WriteString(span.Text())
+			text.WriteString(" ")
+		})
+		audio = append(audio, findAudioFromText(text.String())...)
+		y := findYearFromText(text.String(), title)
+		if y != "" {
+			year = y
+		}
+		size = append(size, findSizesFromText(text.String())...)
+	})
+
+	// TODO: find any link from imdb
+	imdbLink := ""
+
+	size = stableUniq(size)
+
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
+
+	// for each magnet link, create a new indexed torrent
+	for it, magnetLink := range magnetLinks {
+		it := it
+		go func(it int, magnetLink string) {
+			magnet, err := magnet.ParseMagnetUri(magnetLink)
+			if err != nil {
+				fmt.Println(err)
+			}
+			releaseTitle := strings.TrimSpace(magnet.DisplayName)
+			// url decode the title
+			releaseTitle, err = url.QueryUnescape(releaseTitle)
+			if err != nil {
+				fmt.Println(err)
+				releaseTitle = strings.TrimSpace(magnet.DisplayName)
+			}
+			infoHash := magnet.InfoHash.String()
+			trackers := magnet.Trackers
+			for i, tracker := range trackers {
+				unescapedTracker, err := url.QueryUnescape(tracker)
+				if err != nil {
+					fmt.Println(err)
+				}
+				trackers[i] = strings.TrimSpace(unescapedTracker)
+			}
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
+
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+			if err != nil {
+				fmt.Println(err)
+			}
+
+			title := processTitle(title, magnetAudio)
+
+			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
+			var mySize string
+			if len(size) == len(magnetLinks) {
+				mySize = size[it]
+			}
+
+			ixt := schema.IndexedTorrent{
+				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				OriginalTitle: title,
+				Details:       link,
+				Year:          year,
+				IMDB:          imdbLink,
+				Audio:         magnetAudio,
+				MagnetLink:    magnetLink,
+				Date:          date,
+				InfoHash:      infoHash,
+				Trackers:      trackers,
+				LeechCount:    peer,
+				SeedCount:     seed,
+				Size:          mySize,
+			}
+			chanIndexedTorrent <- ixt
+		}(it, magnetLink)
+	}
+
+	for i := 0; i < len(magnetLinks); i++ {
+		it := <-chanIndexedTorrent
+		indexedTorrents = append(indexedTorrents, it)
+	}
+
+	return indexedTorrents, nil
+}

api/torrent_dos_filmes.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
-	"regexp"
 	"slices"
 	"strings"
 	"time"
@@ -33,14 +32,17 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
 	}()
 
 	ctx := r.Context()
-	// supported query params: q, season, episode, filter_results
+	// supported query params: q, season, episode, page, filter_results
 	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
 
 	// URL encode query param
 	q = url.QueryEscape(q)
 	url := torrent_dos_filmes.URL
 	if q != "" {
 		url = fmt.Sprintf("%s%s%s", url, torrent_dos_filmes.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf("%spage/%s", url, page)
 	}
 
 	fmt.Println("URL:>", url)
@@ -74,9 +76,9 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []IndexedTorrent)
+	var itChan = make(chan []schema.IndexedTorrent)
 	var errChan = make(chan error)
-	indexedTorrents := []IndexedTorrent{}
+	indexedTorrents := []schema.IndexedTorrent{}
 	for _, link := range links {
 		go func(link string) {
 			torrents, err := getTorrentsTorrentDosFilmes(ctx, i, link)
@@ -106,16 +108,21 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
 
 	// remove the ones with zero similarity
 	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
+		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
 			return it.Similarity > 0
 		})
 	}
 
 	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
+	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
 		return int((j.Similarity - i.Similarity) * 1000)
 	})
 
+	// send to search index
+	go func() {
+		_ = i.search.IndexTorrents(indexedTorrents)
+	}()
+
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(Response{
 		Results: indexedTorrents,
@@ -126,8 +133,8 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
 	}
 }
 
-func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
+func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
-	var indexedTorrents []IndexedTorrent
+	var indexedTorrents []schema.IndexedTorrent
 	doc, err := getDocument(ctx, i, link)
 	if err != nil {
 		return nil, err
@@ -136,7 +143,7 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
 	article := doc.Find("article")
 	title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1)
 	textContent := article.Find("div.content")
-	date := getPublishedDateTDF(doc)
+	date := getPublishedDateFromMeta(doc)
 	magnets := textContent.Find("a[href^=\"magnet\"]")
 	var magnetLinks []string
 	magnets.Each(func(i int, s *goquery.Selection) {
@@ -178,16 +185,15 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
 	imdbLink := ""
 	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
 		link, _ := s.Attr("href")
-		re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
+		_imdbLink, err := getIMDBLink(link)
-		matches := re.FindStringSubmatch(link)
+		if err == nil {
-		if len(matches) > 0 {
+			imdbLink = _imdbLink
-			imdbLink = matches[0]
 		}
 	})
 
 	size = stableUniq(size)
 
-	var chanIndexedTorrent = make(chan IndexedTorrent)
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
 
 	// for each magnet link, create a new indexed torrent
 	for it, magnetLink := range magnetLinks {
@@ -200,19 +206,7 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
 			releaseTitle := magnet.DisplayName
 			infoHash := magnet.InfoHash.String()
 			trackers := magnet.Trackers
-			magnetAudio := []schema.Audio{}
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
-			if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
-				magnetAudio = append(magnetAudio, audio...)
-			} else if len(audio) > 1 {
-				// remove portuguese audio, and append to magnetAudio
-				for _, a := range audio {
-					if a != schema.AudioPortuguese {
-						magnetAudio = append(magnetAudio, a)
-					}
-				}
-			} else {
-				magnetAudio = append(magnetAudio, audio...)
-			}
 
 			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
 			if err != nil {
@@ -227,7 +221,7 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
 				mySize = size[it]
 			}
 
-			ixt := IndexedTorrent{
+			ixt := schema.IndexedTorrent{
 				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
 				OriginalTitle: title,
 				Details:       link,
@@ -253,15 +247,3 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
 
 	return indexedTorrents, nil
 }
-
-func getPublishedDateTDF(document *goquery.Document) time.Time {
-	var date time.Time
-	//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">
-	datePublished := strings.TrimSpace(document.Find("meta[property=\"article:published_time\"]").AttrOr("content", ""))
-
-	if datePublished != "" {
-		date, _ = time.Parse(time.RFC3339, datePublished)
-	}
-
-	return date
-}

cache/redis.go

@@ -9,13 +9,14 @@ import (
 	"github.com/redis/go-redis/v9"
 )
 
-var (
+const (
 	DefaultExpiration      = 24 * time.Hour * 7 // 7 days
 	IndexerComandoTorrents = "indexer:comando_torrents"
 )
 
 type Redis struct {
-	client *redis.Client
+	client            *redis.Client
+	defaultExpiration time.Duration
 }
 
 func NewRedis() *Redis {
@@ -28,15 +29,20 @@ func NewRedis() *Redis {
 			Addr:     fmt.Sprintf("%s:6379", redisHost),
 			Password: "",
 		}),
+		defaultExpiration: DefaultExpiration,
 	}
 }
 
+func (r *Redis) SetDefaultExpiration(expiration time.Duration) {
+	r.defaultExpiration = expiration
+}
+
 func (r *Redis) Get(ctx context.Context, key string) ([]byte, error) {
 	return r.client.Get(ctx, key).Bytes()
 }
 
 func (r *Redis) Set(ctx context.Context, key string, value []byte) error {
-	return r.client.Set(ctx, key, value, DefaultExpiration).Err()
+	return r.client.Set(ctx, key, value, r.defaultExpiration).Err()
 }
 
 func (r *Redis) SetWithExpiration(ctx context.Context, key string, value []byte, expiration time.Duration) error {

docker-compose.yml

@@ -11,6 +11,8 @@ services:
       - indexer
     environment:
       - REDIS_HOST=redis
+      - MEILISEARCH_ADDRESS=http://meilisearch:7700
+      - MEILISEARCH_KEY=my-secret-key
       - FLARESOLVERR_ADDRESS=http://flaresolverr:8191
   
   redis:
@@ -20,5 +22,17 @@ services:
     networks:
       - indexer
 
+  # This container is not necessary for the indexer to work,
+  # deploy if you want to use the search feature
+  meilisearch: 
+    image: getmeili/meilisearch:latest
+    container_name: meilisearch
+    restart: unless-stopped
+    networks:
+      - indexer
+    environment:
+      - MEILI_NO_ANALYTICS=true
+      - MEILI_MASTER_KEY=my-secret-key
+
 networks:
   indexer:

go.mod

@@ -12,7 +12,6 @@ require (
 	github.com/prometheus/client_model v0.6.0 // indirect
 	github.com/prometheus/common v0.50.0 // indirect
 	github.com/prometheus/procfs v0.13.0 // indirect
-	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 )
@@ -21,4 +20,6 @@ require (
 	github.com/PuerkitoBio/goquery v1.9.1
 	github.com/hbollon/go-edlib v1.6.0
 	github.com/prometheus/client_golang v1.19.0
+	github.com/xhit/go-str2duration/v2 v2.1.0
+	golang.org/x/net v0.22.0
 )

go.sum

@@ -28,6 +28,8 @@ github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGK
 github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
 github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
 github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
+github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

main.go

@@ -8,27 +8,55 @@ import (
 	handler "github.com/felipemarinho97/torrent-indexer/api"
 	"github.com/felipemarinho97/torrent-indexer/cache"
 	"github.com/felipemarinho97/torrent-indexer/monitoring"
+	"github.com/felipemarinho97/torrent-indexer/public"
 	"github.com/felipemarinho97/torrent-indexer/requester"
+	meilisearch "github.com/felipemarinho97/torrent-indexer/search"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	str2duration "github.com/xhit/go-str2duration/v2"
 )
 
 func main() {
 	redis := cache.NewRedis()
+	searchIndex := meilisearch.NewSearchIndexer(os.Getenv("MEILISEARCH_ADDRESS"), os.Getenv("MEILISEARCH_KEY"), "torrents")
 	metrics := monitoring.NewMetrics()
 	metrics.Register()
 
 	flaresolverr := requester.NewFlareSolverr(os.Getenv("FLARESOLVERR_ADDRESS"), 60000)
 	req := requester.NewRequester(flaresolverr, redis)
-	indexers := handler.NewIndexers(redis, metrics, req)
+
+	// get shot-lived and long-lived cache expiration from env
+	shortLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("SHORT_LIVED_CACHE_EXPIRATION"))
+	if err == nil {
+		fmt.Printf("Setting short-lived cache expiration to %s\n", shortLivedCacheExpiration)
+		req.SetShortLivedCacheExpiration(shortLivedCacheExpiration)
+	}
+	longLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("LONG_LIVED_CACHE_EXPIRATION"))
+	if err == nil {
+		fmt.Printf("Setting long-lived cache expiration to %s\n", longLivedCacheExpiration)
+		redis.SetDefaultExpiration(longLivedCacheExpiration)
+	} else {
+		fmt.Println(err)
+	}
+
+	indexers := handler.NewIndexers(redis, metrics, req, searchIndex)
+	search := handler.NewMeilisearchHandler(searchIndex)
 
 	indexerMux := http.NewServeMux()
 	metricsMux := http.NewServeMux()
 
 	indexerMux.HandleFunc("/", handler.HandlerIndex)
-	indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
-	indexerMux.HandleFunc("/indexers/torrent-dos-filmes", indexers.HandlerTorrentDosFilmesIndexer)
 	indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
+	indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
+	indexerMux.HandleFunc("/indexers/comandohds", indexers.HandlerComandoHDsIndexer)
+	indexerMux.HandleFunc("/indexers/rede_torrent", indexers.HandlerRedeTorrentIndexer)
+	indexerMux.HandleFunc("/indexers/starck-filmes", indexers.HandlerStarckFilmesIndexer)
+	indexerMux.HandleFunc("/indexers/torrent-dos-filmes", indexers.HandlerTorrentDosFilmesIndexer)
 	indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer)
+	indexerMux.HandleFunc("/search", search.SearchTorrentHandler)
+	indexerMux.HandleFunc("/search/health", search.HealthHandler)
+	indexerMux.HandleFunc("/search/stats", search.StatsHandler)
+	indexerMux.Handle("/ui/", http.StripPrefix("/ui/", http.FileServer(http.FS(public.UIFiles))))
 
 	metricsMux.Handle("/metrics", promhttp.Handler())
 
@@ -38,8 +66,14 @@ func main() {
 			panic(err)
 		}
 	}()
-	fmt.Println("Server listening on :7006")
+
-	err := http.ListenAndServe(":7006", indexerMux)
+	port := os.Getenv("PORT")
+	if port == "" {
+		port = "7006"
+	}
+
+	fmt.Printf("Server listening on :%s\n", port)
+	err = http.ListenAndServe(":"+port, indexerMux)
 	if err != nil {
 		panic(err)
 	}

public/index.go

@@ -0,0 +1,6 @@
+package public
+
+import "embed"
+
+//go:embed *
+var UIFiles embed.FS

public/index.html

@@ -0,0 +1,226 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Torrent Indexer</title>
+  <script src="https://cdn.tailwindcss.com"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@heroicons/react/solid@2.0.0/dist/index.umd.js"></script>
+</head>
+
+<body class="bg-gray-900 text-white font-sans min-h-screen flex flex-col">
+  <div class="container mx-auto p-6 flex-grow">
+    <!-- Header -->
+    <header class="text-center mb-10">
+      <h1 class="text-4xl font-bold text-blue-400">Torrent Indexer 🇧🇷</h1>
+      <p class="text-gray-400 mt-2">Find torrents with detailed information from torrent-indexer cache</p>
+    </header>
+
+    <!-- Search Bar -->
+    <div class="flex justify-center mb-10">
+      <input id="search-query" type="text" placeholder="Enter search query"
+        class="w-full max-w-lg px-4 py-2 rounded-md border border-gray-600 bg-gray-800 text-white focus:ring focus:ring-blue-500">
+      <button id="search-btn"
+        class="ml-4 px-6 py-2 bg-blue-600 hover:bg-blue-700 rounded-md font-bold text-white">Search</button>
+    </div>
+    
+    <!-- Results Section -->
+    <div id="results" class="space-y-6 mb-10">
+      <!-- Dynamic content will be injected here -->
+    </div>
+
+    <!-- Health Warning -->
+    <div id="health-warning" class="hidden mb-6 p-4 bg-yellow-800 border border-yellow-600 rounded-lg">
+      <div class="flex items-center gap-2">
+        <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5 text-yellow-400" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+          <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 9v2m0 4h.01m-9 4h18a2 2 0 002-2V7a2 2 0 00-2-2H3a2 2 0 00-2 2v10a2 2 0 002 2z" />
+        </svg>
+        <span class="font-bold text-yellow-400">Service Warning</span>
+      </div>
+      <p class="text-yellow-200 mt-2">Search functionality may be disabled or experiencing issues. Please try again later.</p>
+    </div>
+
+  </div>
+  <!-- Stats Section -->
+  <div id="database-statistics" class="stats-info mt-auto mb-1 p-3 rounded text-center">
+    <span id="torrentStats" class="text-gray-400">Loading stats...</span>
+  </div>
+
+  <script>
+    // Global variables
+    let serviceHealthy = true;
+
+    // Function to check service health
+    async function checkHealth() {
+      try {
+        const response = await fetch('/search/health');
+        const health = await response.json();
+        
+        if (response.status === 503 || health.status === 'unhealthy') {
+          serviceHealthy = false;
+          showHealthWarning();
+          hideDatabaseStatistics();
+        } else if (health.status === 'degraded') {
+          serviceHealthy = true; // Still operational
+          showHealthWarning(); // But show warning
+        } else {
+          serviceHealthy = true;
+          hideHealthWarning();
+        }
+      } catch (error) {
+        serviceHealthy = false;
+        showHealthWarning();
+        console.error('Health check failed:', error);
+      }
+    }
+
+    // Function to show health warning
+    function showHealthWarning() {
+      document.getElementById('health-warning').classList.remove('hidden');
+    }
+
+    // Function to hide health warning
+    function hideHealthWarning() {
+      document.getElementById('health-warning').classList.add('hidden');
+    }
+
+    function hideDatabaseStatistics() {
+      document.getElementById('database-statistics').classList.add('hidden');
+    }
+
+    // Function to load stats
+    async function loadStats() {
+      try {
+        const response = await fetch('/search/stats');
+        if (response.ok) {
+          const stats = await response.json();
+          const statsElement = document.getElementById('torrentStats');
+          
+          const formattedStats = `
+              <span class="text-sm text-gray-500">
+                <span class="text-green-400 font-medium">${stats.numberOfDocuments?.toLocaleString()+'+' || 'N/A'}</span> indexed torrents!
+              </span>
+          `;
+          
+          statsElement.innerHTML = formattedStats;
+        } else {
+          throw new Error('Failed to load stats');
+        }
+      } catch (error) {
+        hideDatabaseStatistics();
+        console.error('Stats loading failed:', error);
+      }
+    }
+
+    // Function to render a single torrent result
+    function renderTorrent(torrent) {
+      return `
+        <div class="p-6 bg-gray-800 rounded-lg shadow-md flex flex-col md:flex-row gap-6">
+          <!-- Torrent Title and Details -->
+          <div class="flex-grow">
+            <h2 class="text-2xl font-bold text-blue-400 flex items-center gap-2">
+              <span>${torrent.title}</span>
+              <span class="text-sm text-gray-400">(${torrent.year})</span>
+            </h2>
+            <p class="text-gray-500 italic mt-1">${torrent.original_title}</p>
+            <div class="mt-4 grid grid-cols-1 sm:grid-cols-2 gap-4">
+              <p><strong>Audio:</strong> ${torrent.audio.join(', ')}</p>
+              <p><strong>Size:</strong> ${torrent.size}</p>
+              <p><strong>Seeds:</strong> ${torrent.seed_count} | <strong>Leeches:</strong> ${torrent.leech_count}</p>
+              <p><strong>Info Hash:</strong> <span class="text-sm break-all text-gray-300">${torrent.info_hash}</span></p>
+            </div>
+          </div>
+
+          <!-- Actions -->
+          <div class="flex flex-col justify-between items-start md:items-end">
+            <div>
+              <a href="${torrent.imdb}" target="_blank"
+                class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium">
+                <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                  <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                    d="M8 16l-4-4m0 0l4-4m-4 4h16" />
+                </svg>
+                View on IMDB
+              </a>
+              <a href="${torrent.details}" target="_blank"
+                class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium mt-2">
+                <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                  <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                    d="M13 16h-1v-4h-.01M9 20h6a2 2 0 002-2v-5a2 2 0 00-2-2h-3.5a2 2 0 00-1.85 1.19M13 10V6a3 3 0 00-6 0v4" />
+                </svg>
+                View Details
+              </a>
+            </div>
+            <a href="${torrent.magnet_link}" target="_blank"
+              class="px-4 py-2 bg-green-600 hover:bg-green-700 text-white font-bold rounded-md flex items-center gap-2 mt-4">
+              <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                  d="M9 17v-6m6 6v-6m-6 6l-2-2m8 0l2-2M5 9l7-7 7 7" />
+              </svg>
+              Download Magnet
+            </a>
+          </div>
+        </div>
+      `;
+    }
+
+    // Handle search
+    async function onSearch() {
+      const query = document.getElementById('search-query').value.trim();
+      if (!query) {
+        alert('Please enter a search query!');
+        return;
+      }
+
+      try {
+        const response = await fetch(`/search?q=${encodeURIComponent(query)}`);
+        if (!response.ok) {
+          throw new Error('Search failed');
+        }
+
+        const results = await response.json();
+        const resultsContainer = document.getElementById('results');
+        
+        if (results.length === 0) {
+          resultsContainer.innerHTML = `
+            <div class="p-6 bg-gray-800 rounded-lg shadow-md text-center">
+              <p class="text-xl font-bold text-gray-400">No results found</p>
+              <p class="text-gray-500 mt-2">Try different search terms or check spelling</p>
+            </div>
+          `;
+        } else {
+          resultsContainer.innerHTML = results.map(renderTorrent).join('');
+        }
+      } catch (error) {
+        document.getElementById('results').innerHTML = `
+          <div class="p-6 bg-red-800 rounded-lg shadow-md text-center">
+            <p class="text-xl font-bold text-red-400">Error fetching search results</p>
+            <p class="text-gray-400 mt-2">Please try again later.</p>
+          </div>
+        `;
+        console.error(error);
+      }
+    }
+    
+    // Event listeners
+    document.getElementById('search-btn').addEventListener('click', onSearch);
+    document.getElementById('search-query').addEventListener('keypress', (e) => {
+      if (e.key === 'Enter') {
+        onSearch();
+      }
+    });
+
+    // Initialize page
+    document.addEventListener('DOMContentLoaded', async () => {
+      await checkHealth();
+      await loadStats();
+      
+      // Refresh health and stats periodically
+      setInterval(checkHealth, 30000); // Check health every 30 seconds
+      setInterval(loadStats, 60000);   // Update stats every minute
+    });
+  </script>
+</body>
+
+</html>

requester/flaresolverr.go

@@ -10,6 +10,8 @@ import (
 	"net/url"
 	"strings"
 	"sync"
+
+	"github.com/felipemarinho97/torrent-indexer/utils"
 )
 
 type FlareSolverr struct {
@@ -243,8 +245,15 @@ func (f *FlareSolverr) Get(_url string) (io.ReadCloser, error) {
 		return nil, fmt.Errorf("under attack")
 	}
 
+	// check if the response is valid HTML
+	if !utils.IsValidHTML(response.Solution.Response) {
+		fmt.Printf("[FlareSolverr] Invalid HTML response from %s\n", _url)
+		response.Solution.Response = ""
+	}
+
 	// If the response body is empty but cookies are present, make a new request
 	if response.Solution.Response == "" && len(response.Solution.Cookies) > 0 {
+		fmt.Printf("[FlareSolverr] Making a new request to %s with cookies\n", _url)
 		// Create a new request with cookies
 		client := &http.Client{}
 		cookieJar, err := cookiejar.New(&cookiejar.Options{})
@@ -268,13 +277,22 @@ func (f *FlareSolverr) Get(_url string) (io.ReadCloser, error) {
 			return nil, err
 		}
 
+		// use the same user returned by the FlareSolverr
+		secondReq.Header.Set("User-Agent", response.Solution.UserAgent)
+
 		secondResp, err := client.Do(secondReq)
 		if err != nil {
 			return nil, err
 		}
 
+		respByte := new(bytes.Buffer)
+		_, err = respByte.ReadFrom(secondResp.Body)
+		if err != nil {
+			return nil, err
+		}
+
 		// Return the body of the second request
-		return secondResp.Body, nil
+		return io.NopCloser(bytes.NewReader(respByte.Bytes())), nil
 	}
 
 	// Return the original response body

requester/requester.go

@@ -10,23 +10,28 @@ import (
 	"time"
 
 	"github.com/felipemarinho97/torrent-indexer/cache"
+	"github.com/felipemarinho97/torrent-indexer/utils"
 )
 
 const (
-	shortLivedCacheExpiration = 30 * time.Minute
+	cacheKey = "shortLivedCache"
-	cacheKey                  = "shortLivedCache"
 )
 
 var challangeRegex = regexp.MustCompile(`(?i)(just a moment|cf-chl-bypass|under attack)`)
 
 type Requster struct {
-	fs         *FlareSolverr
+	fs                        *FlareSolverr
-	c          *cache.Redis
+	c                         *cache.Redis
-	httpClient *http.Client
+	httpClient                *http.Client
+	shortLivedCacheExpiration time.Duration
 }
 
 func NewRequester(fs *FlareSolverr, c *cache.Redis) *Requster {
-	return &Requster{fs: fs, httpClient: &http.Client{}, c: c}
+	return &Requster{fs: fs, httpClient: &http.Client{}, c: c, shortLivedCacheExpiration: 30 * time.Minute}
+}
+
+func (i *Requster) SetShortLivedCacheExpiration(expiration time.Duration) {
+	i.shortLivedCacheExpiration = expiration
 }
 
 func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser, error) {
@@ -73,9 +78,9 @@ func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser,
 		fmt.Printf("request served from plain client: %s\n", url)
 	}
 
-	// save response to cache if it's not a challange and body is not empty
+	// save response to cache if it's not a challange, body is not empty and is valid HTML
-	if !hasChallange(bodyByte) && len(bodyByte) > 0 {
+	if !hasChallange(bodyByte) && len(bodyByte) > 0 && utils.IsValidHTML(string(bodyByte)) {
-		err = i.c.SetWithExpiration(ctx, key, bodyByte, shortLivedCacheExpiration)
+		err = i.c.SetWithExpiration(ctx, key, bodyByte, i.shortLivedCacheExpiration)
 		if err != nil {
 			fmt.Printf("failed to save response to cache: %v\n", err)
 		}

schema/audio.go

@@ -33,6 +33,10 @@ const (
 	AudioThai2       = "Tailandes"
 	AudioTurkish     = "Turco"
 	AudioHindi       = "Hindi"
+	AudioFarsi       = "Persa"
+	AudioMalay       = "Malaio"
+	AudioDutch       = "Holandês"
+	AudioDutch2      = "Holandes"
 )
 
 var AudioList = []Audio{
@@ -64,6 +68,10 @@ var AudioList = []Audio{
 	AudioThai2,
 	AudioTurkish,
 	AudioHindi,
+	AudioFarsi,
+	AudioMalay,
+	AudioDutch,
+	AudioDutch2,
 }
 
 func (a Audio) String() string {
@@ -137,6 +145,14 @@ func (a Audio) toTag() string {
 		return "tur"
 	case AudioHindi:
 		return "hin"
+	case AudioFarsi:
+		return "fas"
+	case AudioMalay:
+		return "msa"
+	case AudioDutch:
+		return "nld"
+	case AudioDutch2:
+		return "nld"
 	default:
 		return ""
 	}

schema/indexed_torrent.go

@@ -0,0 +1,20 @@
+package schema
+
+import "time"
+
+type IndexedTorrent struct {
+	Title         string    `json:"title"`
+	OriginalTitle string    `json:"original_title"`
+	Details       string    `json:"details"`
+	Year          string    `json:"year"`
+	IMDB          string    `json:"imdb"`
+	Audio         []Audio   `json:"audio"`
+	MagnetLink    string    `json:"magnet_link"`
+	Date          time.Time `json:"date"`
+	InfoHash      string    `json:"info_hash"`
+	Trackers      []string  `json:"trackers"`
+	Size          string    `json:"size"`
+	LeechCount    int       `json:"leech_count"`
+	SeedCount     int       `json:"seed_count"`
+	Similarity    float32   `json:"similarity"`
+}

search/meilisearch.go

@@ -0,0 +1,223 @@
+package meilisearch
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/felipemarinho97/torrent-indexer/schema"
+)
+
+// SearchIndexer integrates with Meilisearch to index and search torrent items.
+type SearchIndexer struct {
+	Client    *http.Client
+	BaseURL   string
+	APIKey    string
+	IndexName string
+}
+
+// IndexStats represents statistics about the Meilisearch index
+type IndexStats struct {
+	NumberOfDocuments int64            `json:"numberOfDocuments"`
+	IsIndexing        bool             `json:"isIndexing"`
+	FieldDistribution map[string]int64 `json:"fieldDistribution"`
+}
+
+// HealthStatus represents the health status of Meilisearch
+type HealthStatus struct {
+	Status string `json:"status"`
+}
+
+// NewSearchIndexer creates a new instance of SearchIndexer.
+func NewSearchIndexer(baseURL, apiKey, indexName string) *SearchIndexer {
+	return &SearchIndexer{
+		Client:    &http.Client{Timeout: 10 * time.Second},
+		BaseURL:   baseURL,
+		APIKey:    apiKey,
+		IndexName: indexName,
+	}
+}
+
+// IndexTorrent indexes a single torrent item in Meilisearch.
+func (t *SearchIndexer) IndexTorrent(torrent schema.IndexedTorrent) error {
+	url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
+
+	torrentWithKey := struct {
+		Hash string `json:"id"`
+		schema.IndexedTorrent
+	}{
+		Hash:           torrent.InfoHash,
+		IndexedTorrent: torrent,
+	}
+
+	jsonData, err := json.Marshal(torrentWithKey)
+	if err != nil {
+		return fmt.Errorf("failed to marshal torrent data: %w", err)
+	}
+
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if t.APIKey != "" {
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
+	}
+
+	resp, err := t.Client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	return nil
+}
+
+func (t *SearchIndexer) IndexTorrents(torrents []schema.IndexedTorrent) error {
+	url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
+
+	torrentsWithKey := make([]struct {
+		Hash string `json:"id"`
+		schema.IndexedTorrent
+	}, 0, len(torrents))
+	for _, torrent := range torrents {
+		torrentWithKey := struct {
+			Hash string `json:"id"`
+			schema.IndexedTorrent
+		}{
+			Hash:           torrent.InfoHash,
+			IndexedTorrent: torrent,
+		}
+		torrentsWithKey = append(torrentsWithKey, torrentWithKey)
+	}
+
+	jsonData, err := json.Marshal(torrentsWithKey)
+	if err != nil {
+		return fmt.Errorf("failed to marshal torrent data: %w", err)
+	}
+
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if t.APIKey != "" {
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
+	}
+
+	resp, err := t.Client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	return nil
+}
+
+// SearchTorrent searches indexed torrents in Meilisearch based on the query.
+func (t *SearchIndexer) SearchTorrent(query string, limit int) ([]schema.IndexedTorrent, error) {
+	url := fmt.Sprintf("%s/indexes/%s/search", t.BaseURL, t.IndexName)
+	requestBody := map[string]string{
+		"q": query,
+	}
+	jsonData, err := json.Marshal(requestBody)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal search query: %w", err)
+	}
+
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if t.APIKey != "" {
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
+	}
+
+	resp, err := t.Client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, fmt.Errorf("search failed: %s", body)
+	}
+
+	var result struct {
+		Hits []schema.IndexedTorrent `json:"hits"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, fmt.Errorf("failed to parse search response: %w", err)
+	}
+
+	return result.Hits, nil
+}
+
+// GetStats retrieves statistics about the Meilisearch index including document count.
+// This method can be used for health checks and monitoring.
+func (t *SearchIndexer) GetStats() (*IndexStats, error) {
+	url := fmt.Sprintf("%s/indexes/%s/stats", t.BaseURL, t.IndexName)
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	if t.APIKey != "" {
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
+	}
+
+	resp, err := t.Client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, fmt.Errorf("failed to get stats: status %d, body: %s", resp.StatusCode, body)
+	}
+
+	var stats IndexStats
+	if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
+		return nil, fmt.Errorf("failed to parse stats response: %w", err)
+	}
+
+	return &stats, nil
+}
+
+// IsHealthy checks if Meilisearch is available and responsive.
+// Returns true if the service is healthy, false otherwise.
+func (t *SearchIndexer) IsHealthy() bool {
+	url := fmt.Sprintf("%s/health", t.BaseURL)
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return false
+	}
+
+	// Use a shorter timeout for health checks
+	client := &http.Client{Timeout: 5 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	return resp.StatusCode == http.StatusOK
+}
+
+// GetDocumentCount returns the number of indexed documents.
+// This is a convenience method that extracts just the document count from stats.
+func (t *SearchIndexer) GetDocumentCount() (int64, error) {
+	stats, err := t.GetStats()
+	if err != nil {
+		return 0, err
+	}
+	return stats.NumberOfDocuments, nil
+}

utils/decoder.go

@@ -0,0 +1,28 @@
+package utils
+
+import (
+	"encoding/base64"
+	"html"
+)
+
+func DecodeAdLink(encodedStr string) (string, error) {
+	reversed := reverseString(encodedStr)
+
+	decodedBytes, err := base64.StdEncoding.DecodeString(reversed)
+	if err != nil {
+		return "", err
+	}
+
+	htmlUnescaped := html.UnescapeString(string(decodedBytes))
+
+	return htmlUnescaped, nil
+}
+
+// Helper function to reverse a string
+func reverseString(s string) string {
+	runes := []rune(s)
+	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
+		runes[i], runes[j] = runes[j], runes[i]
+	}
+	return string(runes)
+}

utils/util.go

@@ -1,5 +1,10 @@
 package utils
 
+import (
+	"strings"
+	"golang.org/x/net/html"
+)
+
 func Filter[A any](arr []A, f func(A) bool) []A {
 	var res []A
 	res = make([]A, 0)
@@ -10,3 +15,9 @@ func Filter[A any](arr []A, f func(A) bool) []A {
 	}
 	return res
 }
+
+func IsValidHTML(input string) bool {
+	r := strings.NewReader(input)
+	_, err := html.Parse(r)
+	return err == nil
+}