fix: do not cache invalid html

* chg: fix: add bludv url decoding support

* chg: feat: add safeguards against future algorithim changes

README.md

@@ -11,6 +11,8 @@ Visit [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org
 - [comando-torrents](https://comando.la/)
 - [bludv](https://bludvfilmes.tv/)
 - [torrent-dos-filmes](https://torrentdosfilmes.se/)
+- [starck-filmes](https://www.starckfilmes.online/)
+- [comandohds](https://comandohds.org/)
 
 ## Deploy
 
@@ -113,7 +115,91 @@ search:
 # json engine n/a
 ```
 
-If you have more tips on how to integrate with other torrent API clients like Prowlarr, please open a PR.
+## Integrating with Prowlarr
+
+You can integrate this indexer with Prowlarr by adding a custom definition. See [Adding a custom YML definition](https://wiki.servarr.com/prowlarr/indexers#adding-a-custom-yml-definition).
+
+```yaml
+---
+---
+id: torrent-indexer
+name: Torrent Indexer
+description: "Indexing Brazilian Torrent websites into structured data. github.com/felipemarinho97/torrent-indexer"
+language: pt-BR
+type: public
+encoding: UTF-8
+links:
+  - http://localhost:8080/
+
+caps:
+  categories:
+    Movies: Movies
+    TV: TV
+
+  modes:
+    search: [q]
+    tv-search: [q, season]
+    movie-search: [q]
+
+settings:
+  - name: indexer
+    type: select
+    label: Indexer
+    default: bludv
+    options:
+      bludv: BLUDV
+      comando_torrents: Comando Torrents
+      torrent-dos-filmes: Torrent dos Filmes
+
+search:
+  paths:
+    - path: "/indexers/{{ .Config.indexer }}"
+      response:
+        type: json
+  inputs:
+    filter_results: "true"
+    q: "{{ .Keywords }}"
+  keywordsfilters:
+    - name: tolower
+    - name: re_replace
+      args: ["(?i)(S0)(\\d{1,2})$", "temporada $2"]
+    - name: re_replace
+      args: ["(?i)(S)(\\d{1,3})$", "temporada $2"]
+
+  rows:
+    selector: $.results
+    count:
+      selector: $.count
+
+  fields:
+    download:
+      selector: magnet_link
+    title:
+      selector: title
+    description:
+      selector: original_title
+    details:
+      selector: details
+    infohash:
+      selector: info_hash
+    date:
+      selector: date
+    size:
+      selector: size
+    seeders:
+      selector: seed_count
+    leechers:
+      selector: leech_count
+    imdb:
+      selector: imdb
+    category_is_tv_show:
+      selector: title
+      filters:
+        - name: regexp
+          args: "\\b(S\\d+(?:E\\d+)?)\\b"
+    category:
+      text: "{{ if .Result.category_is_tv_show }}TV{{ else }}Movies{{ end }}"
+```
 
 # Warning
 

api/bludv.go

@@ -152,6 +152,30 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In
 		magnetLinks = append(magnetLinks, magnetLink)
 	})
 
+	adwareLinks := textContent.Find("a[href^=\"https://www.seuvideo.xyz\"]")
+	adwareLinks.Each(func(_ int, s *goquery.Selection) {
+		href, _ := s.Attr("href")
+		// extract querysting "id" from url
+		parsedUrl, err := url.Parse(href)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		magnetLink := parsedUrl.Query().Get("id")
+		magnetLinkDecoded, err := utils.DecodeAdLink(magnetLink)
+		if err != nil {
+			fmt.Printf("failed to decode ad link \"%s\": %v\n", href, err)
+			return
+		}
+
+		// if decoded magnet link is indeed a magnet link, append it
+		if strings.HasPrefix(magnetLinkDecoded, "magnet:") {
+			magnetLinks = append(magnetLinks, magnetLinkDecoded)
+		} else {
+			fmt.Printf("WARN: link \"%s\" decoding resulted in non-magnet link: %s\n", href, magnetLinkDecoded)
+		}
+	})
+
 	var audio []schema.Audio
 	var year string
 	var size []string

api/comando_torrents.go

@@ -364,7 +364,7 @@ func findYearFromText(text string, title string) (year string) {
 			year = yearMatch[1]
 		}
 	}
-	return year
+	return strings.TrimSpace(year)
 }
 
 func findAudioFromText(text string) []schema.Audio {

api/comandohds.go

@@ -0,0 +1,263 @@
+package handler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"slices"
+	"strings"
+	"time"
+	"regexp"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/hbollon/go-edlib"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+var comandohds = IndexerMeta{
+	URL:       "https://comandohds.org/",
+	SearchURL: "?s=",
+}
+
+var title_re = regexp.MustCompile(`^[(Filme)|(Série)\s]+`)
+
+func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues("comandohds").Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues("comandohds").Inc()
+	}()
+
+	ctx := r.Context()
+	// supported query params: q, page, filter_results
+	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
+
+	// URL encode query param
+	q = url.QueryEscape(q)
+	url := comandohds.URL
+	if q != "" {
+		url = fmt.Sprintf("%s%s%s", url, comandohds.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf("%spage/%s", url, page)
+	}
+
+	fmt.Println("URL:>", url)
+	resp, err := i.requester.GetDocument(ctx, url)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
+		return
+	}
+	defer resp.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
+		return
+	}
+
+	var links []string
+	doc.Find(".post").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Find("div.title > a").Attr("href")
+		links = append(links, link)
+	})
+
+	var itChan = make(chan []schema.IndexedTorrent)
+	var errChan = make(chan error)
+	indexedTorrents := []schema.IndexedTorrent{}
+	for _, link := range links {
+		go func(link string) {
+			torrents, err := getTorrentsComandoHDs(ctx, i, link)
+			if err != nil {
+				fmt.Println(err)
+				errChan <- err
+			}
+			itChan <- torrents
+		}(link)
+	}
+
+	for i := 0; i < len(links); i++ {
+		select {
+		case torrents := <-itChan:
+			indexedTorrents = append(indexedTorrents, torrents...)
+		case err := <-errChan:
+			fmt.Println(err)
+		}
+	}
+
+	for i, it := range indexedTorrents {
+		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
+		qLower := strings.ToLower(q)
+		splitLength := 2
+		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
+	}
+
+	// remove the ones with zero similarity
+	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
+		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
+			return it.Similarity > 0
+		})
+	}
+
+	// sort by similarity
+	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
+		return int((j.Similarity - i.Similarity) * 1000)
+	})
+
+	// send to search index
+	go func() {
+		_ = i.search.IndexTorrents(indexedTorrents)
+	}()
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: indexedTorrents,
+		Count:   len(indexedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
+func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+	var indexedTorrents []schema.IndexedTorrent
+	doc, err := getDocument(ctx, i, link)
+	if err != nil {
+		return nil, err
+	}
+
+	article := doc.Find("article")
+	title := title_re.ReplaceAllString(article.Find(".main_title > h1").Text(), "")
+	textContent := article.Find("div.content")
+	date := getPublishedDateTDF(doc)
+	magnets := textContent.Find("a[href^=\"magnet\"]")
+	var magnetLinks []string
+	magnets.Each(func(i int, s *goquery.Selection) {
+		magnetLink, _ := s.Attr("href")
+		magnetLinks = append(magnetLinks, magnetLink)
+	})
+
+	var audio []schema.Audio
+	var year string
+	var size []string
+	article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
+		// pattern:
+		// »INFORMAÇÕES«
+		// Titulo Traduzido: O Guerreiro Banido
+		// Titulo Original: 天龍八部之喬峰傳
+		// <picture />: 5.7
+		// Ano de Lançamento: 2023
+		// Gênero: Ação
+		// Formato: MKV
+		// Qualidade: WEB-DL
+		// Idioma: Português | Inglês
+		// Legenda: Português
+		// Tamanho: – GB
+		// Qualidade Áudio e Vídeo: 10
+		// Duração: 130 Min
+		// Servidor: Torrent
+		text := s.Text()
+
+		audio = append(audio, findAudioFromText(text)...)
+		y := findYearFromText(text, title)
+		if y != "" {
+			year = y
+		}
+		size = append(size, findSizesFromText(text)...)
+	})
+
+	// find any link from imdb
+	imdbLink := ""
+	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Attr("href")
+		_imdbLink, err := getIMDBLink(link)
+		if err == nil {
+			imdbLink = _imdbLink
+		}
+	})
+
+	size = stableUniq(size)
+
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
+
+	// for each magnet link, create a new indexed torrent
+	for it, magnetLink := range magnetLinks {
+		it := it
+		go func(it int, magnetLink string) {
+			magnet, err := magnet.ParseMagnetUri(magnetLink)
+			if err != nil {
+				fmt.Println(err)
+			}
+			releaseTitle := magnet.DisplayName
+			infoHash := magnet.InfoHash.String()
+			trackers := magnet.Trackers
+			magnetAudio := []schema.Audio{}
+			if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
+				magnetAudio = append(magnetAudio, audio...)
+			} else if len(audio) > 1 {
+				// remove portuguese audio, and append to magnetAudio
+				for _, a := range audio {
+					if a != schema.AudioPortuguese {
+						magnetAudio = append(magnetAudio, a)
+					}
+				}
+			} else {
+				magnetAudio = append(magnetAudio, audio...)
+			}
+
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+			if err != nil {
+				fmt.Println(err)
+			}
+
+			title := processTitle(title, magnetAudio)
+
+			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
+			var mySize string
+			if len(size) == len(magnetLinks) {
+				mySize = size[it]
+			}
+
+			ixt := schema.IndexedTorrent{
+				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				OriginalTitle: title,
+				Details:       link,
+				Year:          year,
+				IMDB:          imdbLink,
+				Audio:         magnetAudio,
+				MagnetLink:    magnetLink,
+				Date:          date,
+				InfoHash:      infoHash,
+				Trackers:      trackers,
+				LeechCount:    peer,
+				SeedCount:     seed,
+				Size:          mySize,
+			}
+			chanIndexedTorrent <- ixt
+		}(it, magnetLink)
+	}
+
+	for i := 0; i < len(magnetLinks); i++ {
+		it := <-chanIndexedTorrent
+		indexedTorrents = append(indexedTorrents, it)
+	}
+
+	return indexedTorrents, nil
+}

api/index.go

@@ -77,6 +77,28 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
 					},
 				},
 			},
+			"/indexers/comandohds": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"page":        "page number",
+					"description": "Indexer for Comando HDs",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
+				},
+			},
+			"/indexers/starck-filmes": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"page":        "page number",
+					"description": "Indexer for Starck Filmes",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
+				},
+			},
 			"/indexers/manual": []map[string]interface{}{
 				{
 					"method":      "POST",

api/starck_filmes.go

@@ -0,0 +1,253 @@
+package handler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/hbollon/go-edlib"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+var starck_filmes = IndexerMeta{
+	URL:       "https://www.starckfilmes.online/",
+	SearchURL: "?s=",
+}
+
+func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues("starck_filmes").Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues("starck_filmes").Inc()
+	}()
+
+	ctx := r.Context()
+	// supported query params: q, page, filter_results
+	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
+
+	// URL encode query param
+	q = url.QueryEscape(q)
+	url := starck_filmes.URL
+	if q != "" {
+		url = fmt.Sprintf("%s%s%s", url, starck_filmes.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf("%spage/%s", url, page)
+	}
+
+	fmt.Println("URL:>", url)
+	resp, err := i.requester.GetDocument(ctx, url)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
+		return
+	}
+	defer resp.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
+		return
+	}
+
+	var links []string
+	doc.Find(".item").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Find("div.sub-item > a").Attr("href")
+		links = append(links, link)
+	})
+
+	var itChan = make(chan []schema.IndexedTorrent)
+	var errChan = make(chan error)
+	indexedTorrents := []schema.IndexedTorrent{}
+	for _, link := range links {
+		go func(link string) {
+			torrents, err := getTorrentStarckFilmes(ctx, i, link)
+			if err != nil {
+				fmt.Println(err)
+				errChan <- err
+			}
+			itChan <- torrents
+		}(link)
+	}
+
+	for i := 0; i < len(links); i++ {
+		select {
+		case torrents := <-itChan:
+			indexedTorrents = append(indexedTorrents, torrents...)
+		case err := <-errChan:
+			fmt.Println(err)
+		}
+	}
+
+	for i, it := range indexedTorrents {
+		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
+		qLower := strings.ToLower(q)
+		splitLength := 2
+		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
+	}
+
+	// remove the ones with zero similarity
+	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
+		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
+			return it.Similarity > 0
+		})
+	}
+
+	// sort by similarity
+	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
+		return int((j.Similarity - i.Similarity) * 1000)
+	})
+
+	// send to search index
+	go func() {
+		_ = i.search.IndexTorrents(indexedTorrents)
+	}()
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: indexedTorrents,
+		Count:   len(indexedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
+func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+	var indexedTorrents []schema.IndexedTorrent
+	doc, err := getDocument(ctx, i, link)
+	if err != nil {
+		return nil, err
+	}
+
+	post := doc.Find(".post")
+	capa := post.Find(".capa")
+	title := capa.Find(".post-description > h2").Text()
+	post_buttons := post.Find(".post-buttons")
+	magnets := post_buttons.Find("a[href^=\"magnet\"]")
+	var magnetLinks []string
+	magnets.Each(func(i int, s *goquery.Selection) {
+		magnetLink, _ := s.Attr("href")
+		magnetLinks = append(magnetLinks, magnetLink)
+	})
+
+	var audio []schema.Audio
+	var year string
+	var size []string
+	capa.Find(".post-description p").Each(func(i int, s *goquery.Selection) {
+		// pattern:
+		// Nome Original: 28 Weeks Later
+		// Lançamento: 2007
+		// Duração: 1h 40 min
+		// Gênero: Terror, Suspense, Ficção
+		// Formato: MKV
+		// Tamanho: 2.45 GB 
+		// Qualidade de Video: 10
+		// Qualidade do Audio: 10
+		// Idioma: Português | Inglês
+		// Legenda: Português, Inglês, Espanhol
+		var text strings.Builder
+		s.Find("span").Each(func (i int, span *goquery.Selection) {
+			text.WriteString(span.Text())
+			text.WriteString(" ")
+		})
+		fmt.Println(text.String())
+		audio = append(audio, findAudioFromText(text.String())...)
+		y := findYearFromText(text.String(), title)
+		if y != "" {
+			year = y
+		}
+		size = append(size, findSizesFromText(text.String())...)
+	})
+
+	// TODO: find any link from imdb
+	imdbLink := ""
+
+	size = stableUniq(size)
+
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
+
+	// for each magnet link, create a new indexed torrent
+	for it, magnetLink := range magnetLinks {
+		it := it
+		go func(it int, magnetLink string) {
+			magnet, err := magnet.ParseMagnetUri(magnetLink)
+			if err != nil {
+				fmt.Println(err)
+			}
+			releaseTitle := magnet.DisplayName
+			infoHash := magnet.InfoHash.String()
+			trackers := magnet.Trackers
+			magnetAudio := []schema.Audio{}
+			if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
+				magnetAudio = append(magnetAudio, audio...)
+			} else if len(audio) > 1 {
+				// remove portuguese audio, and append to magnetAudio
+				for _, a := range audio {
+					if a != schema.AudioPortuguese {
+						magnetAudio = append(magnetAudio, a)
+					}
+				}
+			} else {
+				magnetAudio = append(magnetAudio, audio...)
+			}
+
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+			if err != nil {
+				fmt.Println(err)
+			}
+
+			title := processTitle(title, magnetAudio)
+
+			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
+			var mySize string
+			if len(size) == len(magnetLinks) {
+				mySize = size[it]
+			}
+
+			ixt := schema.IndexedTorrent{
+				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				OriginalTitle: title,
+				Details:       link,
+				Year:          year,
+				IMDB:          imdbLink,
+				Audio:         magnetAudio,
+				MagnetLink:    magnetLink,
+				Date:		   time.Now(),
+				InfoHash:      infoHash,
+				Trackers:      trackers,
+				LeechCount:    peer,
+				SeedCount:     seed,
+				Size:          mySize,
+			}
+			chanIndexedTorrent <- ixt
+		}(it, magnetLink)
+	}
+
+	for i := 0; i < len(magnetLinks); i++ {
+		it := <-chanIndexedTorrent
+		indexedTorrents = append(indexedTorrents, it)
+	}
+
+	return indexedTorrents, nil
+}

go.mod

@@ -12,7 +12,6 @@ require (
 	github.com/prometheus/client_model v0.6.0 // indirect
 	github.com/prometheus/common v0.50.0 // indirect
 	github.com/prometheus/procfs v0.13.0 // indirect
-	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 )
@@ -22,4 +21,5 @@ require (
 	github.com/hbollon/go-edlib v1.6.0
 	github.com/prometheus/client_golang v1.19.0
 	github.com/xhit/go-str2duration/v2 v2.1.0
+	golang.org/x/net v0.22.0
 )

main.go

@@ -49,6 +49,8 @@ func main() {
 	indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
 	indexerMux.HandleFunc("/indexers/torrent-dos-filmes", indexers.HandlerTorrentDosFilmesIndexer)
 	indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
+	indexerMux.HandleFunc("/indexers/comandohds", indexers.HandlerComandoHDsIndexer)
+	indexerMux.HandleFunc("/indexers/starck-filmes", indexers.HandlerStarckFilmesIndexer)
 	indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer)
 	indexerMux.HandleFunc("/search", search.SearchTorrentHandler)
 	indexerMux.Handle("/ui/", http.StripPrefix("/ui/", http.FileServer(http.FS(public.UIFiles))))

requester/requester.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/felipemarinho97/torrent-indexer/cache"
+	"github.com/felipemarinho97/torrent-indexer/utils"
 )
 
 const (
@@ -77,8 +78,8 @@ func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser,
 		fmt.Printf("request served from plain client: %s\n", url)
 	}
 
-	// save response to cache if it's not a challange and body is not empty
+	// save response to cache if it's not a challange, body is not empty and is valid HTML
-	if !hasChallange(bodyByte) && len(bodyByte) > 0 {
+	if !hasChallange(bodyByte) && len(bodyByte) > 0 && utils.IsValidHTML(string(bodyByte)) {
 		err = i.c.SetWithExpiration(ctx, key, bodyByte, i.shortLivedCacheExpiration)
 		if err != nil {
 			fmt.Printf("failed to save response to cache: %v\n", err)

utils/decoder.go

@@ -0,0 +1,28 @@
+package utils
+
+import (
+	"encoding/base64"
+	"html"
+)
+
+func DecodeAdLink(encodedStr string) (string, error) {
+	reversed := reverseString(encodedStr)
+
+	decodedBytes, err := base64.StdEncoding.DecodeString(reversed)
+	if err != nil {
+		return "", err
+	}
+
+	htmlUnescaped := html.UnescapeString(string(decodedBytes))
+
+	return htmlUnescaped, nil
+}
+
+// Helper function to reverse a string
+func reverseString(s string) string {
+	runes := []rune(s)
+	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
+		runes[i], runes[j] = runes[j], runes[i]
+	}
+	return string(runes)
+}

utils/util.go

@@ -1,6 +1,9 @@
 package utils
 
-import "regexp"
+import (
+	"strings"
+	"golang.org/x/net/html"
+)
 
 func Filter[A any](arr []A, f func(A) bool) []A {
 	var res []A
@@ -14,19 +17,7 @@ func Filter[A any](arr []A, f func(A) bool) []A {
 }
 
 func IsValidHTML(input string) bool {
-	// Check for <!DOCTYPE> declaration (case-insensitive)
+	r := strings.NewReader(input)
-	doctypeRegex := regexp.MustCompile(`(?i)<!DOCTYPE\s+html>`)
+	_, err := html.Parse(r)
-	if !doctypeRegex.MatchString(input) {
+	return err == nil
-		return false
-	}
-
-	// Check for <html> and </html> tags (case-insensitive)
-	htmlTagRegex := regexp.MustCompile(`(?i)<html[\s\S]*?>[\s\S]*?</html>`)
-	if !htmlTagRegex.MatchString(input) {
-		return false
-	}
-
-	// Check for <body> and </body> tags (case-insensitive)
-	bodyTagRegex := regexp.MustCompile(`(?i)<body[\s\S]*?>[\s\S]*?</body>`)
-	return bodyTagRegex.MatchString(input)
 }