Feat/Add post processors + refactor project (#37)

* chg: feat: clean known patterns from title

* chg: refactor: remove duplicated code, and improve maintainability

* chg: feat: add audio tagging post-processor

* chg: refactor: add generic parallelMap function

* chg: refactor: move more function to common locations

* chg: docs: add func docs

api/bludv.go

@@ -6,12 +6,10 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
-	"slices"
 	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/hbollon/go-edlib"
 
 	"github.com/felipemarinho97/torrent-indexer/magnet"
 	"github.com/felipemarinho97/torrent-indexer/schema"
@@ -20,15 +18,19 @@ import (
 )
 
 var bludv = IndexerMeta{
-	URL:       "https://bludv.xyz/",
-	SearchURL: "?s=",
+	Label:       "bludv",
+	URL:         "https://bludv.xyz/",
+	SearchURL:   "?s=",
+	PagePattern: "page/%s",
 }
 
 func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
+	metadata := bludv
+
 	defer func() {
-		i.metrics.IndexerDuration.WithLabelValues("bludv").Observe(time.Since(start).Seconds())
-		i.metrics.IndexerRequests.WithLabelValues("bludv").Inc()
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
 	}()
 
 	ctx := r.Context()
@@ -38,11 +40,11 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 
 	// URL encode query param
 	q = url.QueryEscape(q)
-	url := bludv.URL
+	url := metadata.URL
 	if page != "" {
-		url = fmt.Sprintf("%spage/%s", url, page)
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
 	} else {
-		url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q)
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
 	}
 
 	fmt.Println("URL:>", url)
@@ -53,7 +55,7 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 		if err != nil {
 			fmt.Println(err)
 		}
-		i.metrics.IndexerErrors.WithLabelValues("bludv").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 	defer resp.Close()
@@ -66,7 +68,7 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 			fmt.Println(err)
 		}
 
-		i.metrics.IndexerErrors.WithLabelValues("bludv").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 
@@ -77,57 +79,21 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []schema.IndexedTorrent)
-	var errChan = make(chan error)
-	indexedTorrents := []schema.IndexedTorrent{}
-	for _, link := range links {
-		go func(link string) {
-			torrents, err := getTorrentsBluDV(ctx, i, link)
-			if err != nil {
-				fmt.Println(err)
-				errChan <- err
-			}
-			itChan <- torrents
-		}(link)
-	}
-
-	for i := 0; i < len(links); i++ {
-		select {
-		case torrents := <-itChan:
-			indexedTorrents = append(indexedTorrents, torrents...)
-		case err := <-errChan:
-			fmt.Println(err)
-		}
-	}
-
-	for i, it := range indexedTorrents {
-		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
-		qLower := strings.ToLower(q)
-		splitLength := 2
-		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
-	}
-
-	// remove the ones with zero similarity
-	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
-			return it.Similarity > 0
-		})
-	}
-
-	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
-		return int((j.Similarity - i.Similarity) * 1000)
+	// extract each torrent link
+	indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrentsBluDV(ctx, i, link)
 	})
 
-	// send to search index
-	go func() {
-		_ = i.search.IndexTorrents(indexedTorrents)
-	}()
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
+	}
 
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
 	})
 	if err != nil {
 		fmt.Println(err)
@@ -216,7 +182,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In
 		}
 	})
 
-	size = stableUniq(size)
+	size = utils.StableUniq(size)
 
 	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
 
@@ -247,7 +213,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In
 			}
 
 			ixt := schema.IndexedTorrent{
-				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				Title:         releaseTitle,
 				OriginalTitle: title,
 				Details:       link,
 				Year:          year,

api/comando_torrents.go

@@ -1,15 +1,12 @@
 package handler
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"net/url"
 	"regexp"
-	"slices"
 	"strings"
 	"time"
 
@@ -18,12 +15,13 @@ import (
 	"github.com/felipemarinho97/torrent-indexer/schema"
 	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
 	"github.com/felipemarinho97/torrent-indexer/utils"
-	"github.com/hbollon/go-edlib"
 )
 
 var comando = IndexerMeta{
-	URL:       "https://comando.la/",
-	SearchURL: "?s=",
+	Label:       "comando",
+	URL:         "https://comando.la/",
+	SearchURL:   "?s=",
+	PagePattern: "page/%s",
 }
 
 var replacer = strings.NewReplacer(
@@ -43,9 +41,11 @@ var replacer = strings.NewReplacer(
 
 func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
+	metadata := comando
+
 	defer func() {
-		i.metrics.IndexerDuration.WithLabelValues("comando").Observe(time.Since(start).Seconds())
-		i.metrics.IndexerRequests.WithLabelValues("comando").Inc()
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
 	}()
 
 	ctx := r.Context()
@@ -55,11 +55,11 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 
 	// URL encode query param
 	q = url.QueryEscape(q)
-	url := comando.URL
+	url := metadata.URL
 	if q != "" {
-		url = fmt.Sprintf("%s%s%s", url, comando.SearchURL, q)
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
 	} else if page != "" {
-		url = fmt.Sprintf("%spage/%s", url, page)
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
 	}
 
 	fmt.Println("URL:>", url)
@@ -70,7 +70,7 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 		if err != nil {
 			fmt.Println(err)
 		}
-		i.metrics.IndexerErrors.WithLabelValues("comando").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 	defer resp.Close()
@@ -82,7 +82,7 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 		if err != nil {
 			fmt.Println(err)
 		}
-		i.metrics.IndexerErrors.WithLabelValues("comando").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 
@@ -93,57 +93,21 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []schema.IndexedTorrent)
-	var errChan = make(chan error)
-	indexedTorrents := []schema.IndexedTorrent{}
-	for _, link := range links {
-		go func(link string) {
-			torrents, err := getTorrents(ctx, i, link)
-			if err != nil {
-				fmt.Println(err)
-				errChan <- err
-			}
-			itChan <- torrents
-		}(link)
-	}
-
-	for i := 0; i < len(links); i++ {
-		select {
-		case torrents := <-itChan:
-			indexedTorrents = append(indexedTorrents, torrents...)
-		case err := <-errChan:
-			fmt.Println(err)
-		}
-	}
-
-	for i, it := range indexedTorrents {
-		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
-		qLower := strings.ToLower(q)
-		splitLength := 2
-		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
-	}
-
-	// remove the ones with zero similarity
-	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
-			return it.Similarity > 0
-		})
-	}
-
-	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
-		return int((j.Similarity - i.Similarity) * 1000)
+	// extract each torrent link
+	indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrents(ctx, i, link)
 	})
 
-	// send to search index
-	go func() {
-		_ = i.search.IndexTorrents(indexedTorrents)
-	}()
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
+	}
 
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
 	})
 	if err != nil {
 		fmt.Println(err)
@@ -215,7 +179,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.Indexed
 		}
 	})
 
-	size = stableUniq(size)
+	size = utils.StableUniq(size)
 
 	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
 
@@ -246,7 +210,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.Indexed
 			}
 
 			ixt := schema.IndexedTorrent{
-				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				Title:         releaseTitle,
 				OriginalTitle: title,
 				Details:       link,
 				Year:          year,
@@ -293,38 +257,6 @@ func parseLocalizedDate(datePublished string) (time.Time, error) {
 	return time.Time{}, nil
 }
 
-func stableUniq(s []string) []string {
-	var uniq []map[string]interface{}
-	m := make(map[string]map[string]interface{})
-	for i, v := range s {
-		m[v] = map[string]interface{}{
-			"v": v,
-			"i": i,
-		}
-	}
-	// to order by index
-	for _, v := range m {
-		uniq = append(uniq, v)
-	}
-
-	// sort by index
-	for i := 0; i < len(uniq); i++ {
-		for j := i + 1; j < len(uniq); j++ {
-			if uniq[i]["i"].(int) > uniq[j]["i"].(int) {
-				uniq[i], uniq[j] = uniq[j], uniq[i]
-			}
-		}
-	}
-
-	// get only values
-	var uniqValues []string
-	for _, v := range uniq {
-		uniqValues = append(uniqValues, v["v"].(string))
-	}
-
-	return uniqValues
-}
-
 func processTitle(title string, a []schema.Audio) string {
 	// remove ' - Donwload' from title
 	title = strings.Replace(title, " – Download", "", -1)
@@ -337,38 +269,3 @@ func processTitle(title string, a []schema.Audio) string {
 
 	return title
 }
-
-func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Document, error) {
-	// try to get from redis first
-	docCache, err := i.redis.Get(ctx, link)
-	if err == nil {
-		i.metrics.CacheHits.WithLabelValues("document_body").Inc()
-		fmt.Printf("returning from long-lived cache: %s\n", link)
-		return goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(docCache)))
-	}
-	defer i.metrics.CacheMisses.WithLabelValues("document_body").Inc()
-
-	resp, err := i.requester.GetDocument(ctx, link)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Close()
-
-	body, err := io.ReadAll(resp)
-	if err != nil {
-		return nil, err
-	}
-
-	// set cache
-	err = i.redis.Set(ctx, link, body)
-	if err != nil {
-		fmt.Println(err)
-	}
-
-	doc, err := goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(body)))
-	if err != nil {
-		return nil, err
-	}
-
-	return doc, nil
-}

api/comandohds.go

@@ -7,12 +7,10 @@ import (
 	"net/http"
 	"net/url"
 	"regexp"
-	"slices"
 	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/hbollon/go-edlib"
 
 	"github.com/felipemarinho97/torrent-indexer/magnet"
 	"github.com/felipemarinho97/torrent-indexer/schema"
@@ -21,17 +19,21 @@ import (
 )
 
 var comandohds = IndexerMeta{
-	URL:       "https://comandohds.org/",
-	SearchURL: "?s=",
+	Label:       "comandohds",
+	URL:         "https://comandohds.org/",
+	SearchURL:   "?s=",
+	PagePattern: "page/%s",
 }
 
 var title_re = regexp.MustCompile(`^[(Filme)|(Série)\s]+`)
 
 func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
+	metadata := comandohds
+
 	defer func() {
-		i.metrics.IndexerDuration.WithLabelValues("comandohds").Observe(time.Since(start).Seconds())
-		i.metrics.IndexerRequests.WithLabelValues("comandohds").Inc()
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
 	}()
 
 	ctx := r.Context()
@@ -41,11 +43,11 @@ func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Reques
 
 	// URL encode query param
 	q = url.QueryEscape(q)
-	url := comandohds.URL
+	url := metadata.URL
 	if q != "" {
-		url = fmt.Sprintf("%s%s%s", url, comandohds.SearchURL, q)
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
 	} else if page != "" {
-		url = fmt.Sprintf("%spage/%s", url, page)
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
 	}
 
 	fmt.Println("URL:>", url)
@@ -56,7 +58,7 @@ func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Reques
 		if err != nil {
 			fmt.Println(err)
 		}
-		i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 	defer resp.Close()
@@ -69,7 +71,7 @@ func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Reques
 			fmt.Println(err)
 		}
 
-		i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 
@@ -79,57 +81,21 @@ func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Reques
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []schema.IndexedTorrent)
-	var errChan = make(chan error)
-	indexedTorrents := []schema.IndexedTorrent{}
-	for _, link := range links {
-		go func(link string) {
-			torrents, err := getTorrentsComandoHDs(ctx, i, link)
-			if err != nil {
-				fmt.Println(err)
-				errChan <- err
-			}
-			itChan <- torrents
-		}(link)
-	}
-
-	for i := 0; i < len(links); i++ {
-		select {
-		case torrents := <-itChan:
-			indexedTorrents = append(indexedTorrents, torrents...)
-		case err := <-errChan:
-			fmt.Println(err)
-		}
-	}
-
-	for i, it := range indexedTorrents {
-		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
-		qLower := strings.ToLower(q)
-		splitLength := 2
-		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
-	}
-
-	// remove the ones with zero similarity
-	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
-			return it.Similarity > 0
-		})
-	}
-
-	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
-		return int((j.Similarity - i.Similarity) * 1000)
+	// extract each torrent link
+	indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrentsComandoHDs(ctx, i, link)
 	})
 
-	// send to search index
-	go func() {
-		_ = i.search.IndexTorrents(indexedTorrents)
-	}()
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
+	}
 
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
 	})
 	if err != nil {
 		fmt.Println(err)
@@ -193,7 +159,7 @@ func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]sche
 		}
 	})
 
-	size = stableUniq(size)
+	size = utils.StableUniq(size)
 
 	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
 
@@ -228,7 +194,7 @@ func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]sche
 			}
 
 			ixt := schema.IndexedTorrent{
-				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				Title:         releaseTitle,
 				OriginalTitle: title,
 				Details:       link,
 				Year:          year,

api/common.go

@@ -1,7 +1,10 @@
 package handler
 
 import (
+	"bytes"
+	"context"
 	"fmt"
+	"io"
 	"regexp"
 	"slices"
 	"strings"
@@ -11,6 +14,43 @@ import (
 	"github.com/felipemarinho97/torrent-indexer/schema"
 )
 
+// getDocument retrieves a document from the cache or makes a request to get it.
+// It first checks the Redis cache for the document body.
+func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Document, error) {
+	// try to get from redis first
+	docCache, err := i.redis.Get(ctx, link)
+	if err == nil {
+		i.metrics.CacheHits.WithLabelValues("document_body").Inc()
+		fmt.Printf("returning from long-lived cache: %s\n", link)
+		return goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(docCache)))
+	}
+	defer i.metrics.CacheMisses.WithLabelValues("document_body").Inc()
+
+	resp, err := i.requester.GetDocument(ctx, link)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Close()
+
+	body, err := io.ReadAll(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	// set cache
+	err = i.redis.Set(ctx, link, body)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(body)))
+	if err != nil {
+		return nil, err
+	}
+
+	return doc, nil
+}
+
 func getPublishedDateFromMeta(document *goquery.Document) time.Time {
 	var date time.Time
 	//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">

api/index.go

@@ -13,17 +13,18 @@ import (
 )
 
 type Indexer struct {
-	redis     *cache.Redis
-	metrics   *monitoring.Metrics
-	requester *requester.Requster
-	search    *meilisearch.SearchIndexer
+	redis          *cache.Redis
+	metrics        *monitoring.Metrics
+	requester      *requester.Requster
+	search         *meilisearch.SearchIndexer
+	postProcessors []PostProcessorFunc
 }
 
 type IndexerMeta struct {
-	URL       string
-	SearchURL string
-	// pattern for pagination, e.g. "page/%s"
-	PagePattern string
+	Label       string // Label is used for Prometheus metrics and logging. Must be alphanumeric optionally with underscores.
+	URL         string // URL is the base URL of the indexer, e.g. "https://example.com/"
+	SearchURL   string // SearchURL is the base URL for search queries, e.g. "?s="
+	PagePattern string // PagePattern for pagination, e.g. "page/%s"
 }
 
 type Response struct {
@@ -31,12 +32,22 @@ type Response struct {
 	Count   int                     `json:"count"`
 }
 
+type PostProcessorFunc func(*Indexer, *http.Request, []schema.IndexedTorrent) []schema.IndexedTorrent
+
+var GlobalPostProcessors = []PostProcessorFunc{
+	AddSimilarityCheck,   // Jaccard similarity
+	CleanupTitleWebsites, // Remove website names from titles
+	AppendAudioTags,      // Add (brazilian, eng, etc.) audio tags to titles
+	SendToSearchIndexer,  // Send indexed torrents to Meilisearch
+}
+
 func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster, si *meilisearch.SearchIndexer) *Indexer {
 	return &Indexer{
-		redis:     redis,
-		metrics:   metrics,
-		requester: req,
-		search:    si,
+		redis:          redis,
+		metrics:        metrics,
+		requester:      req,
+		search:         si,
+		postProcessors: GlobalPostProcessors,
 	}
 }
 

api/post_processors.go

@@ -0,0 +1,61 @@
+package handler
+
+import (
+	"fmt"
+	"net/http"
+	"slices"
+	"strings"
+
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+	"github.com/hbollon/go-edlib"
+)
+
+// CleanupTitleWebsites removes unwanted characters from the title
+func CleanupTitleWebsites(_ *Indexer, _ *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
+	for i := range torrents {
+		torrents[i].Title = utils.RemoveKnownWebsites(torrents[i].Title)
+	}
+	return torrents
+}
+
+func AppendAudioTags(_ *Indexer, _ *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
+	for i, it := range torrents {
+		torrents[i].Title = appendAudioISO639_2Code(torrents[i].Title, it.Audio)
+	}
+
+	return torrents
+}
+
+// SendToSearchIndexer sends the indexed torrents to the search indexer
+func SendToSearchIndexer(i *Indexer, _ *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
+	go func() {
+		_ = i.search.IndexTorrents(torrents)
+	}()
+	return torrents
+}
+
+func AddSimilarityCheck(i *Indexer, r *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
+	q := r.URL.Query().Get("q")
+
+	for i, it := range torrents {
+		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
+		qLower := strings.ToLower(q)
+		splitLength := 2
+		torrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
+	}
+
+	// remove the ones with zero similarity
+	if len(torrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
+		torrents = utils.Filter(torrents, func(it schema.IndexedTorrent) bool {
+			return it.Similarity > 0
+		})
+	}
+
+	// sort by similarity
+	slices.SortFunc(torrents, func(i, j schema.IndexedTorrent) int {
+		return int((j.Similarity - i.Similarity) * 1000)
+	})
+
+	return torrents
+}

api/rede_torrent.go

@@ -7,12 +7,10 @@ import (
 	"net/http"
 	"net/url"
 	"regexp"
-	"slices"
 	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/hbollon/go-edlib"
 
 	"github.com/felipemarinho97/torrent-indexer/magnet"
 	"github.com/felipemarinho97/torrent-indexer/schema"
@@ -21,6 +19,7 @@ import (
 )
 
 var rede_torrent = IndexerMeta{
+	Label:       "rede_torrent",
 	URL:         "https://redetorrent.com/",
 	SearchURL:   "index.php?s=",
 	PagePattern: "%s",
@@ -28,9 +27,11 @@ var rede_torrent = IndexerMeta{
 
 func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
+	metadata := rede_torrent
+
 	defer func() {
-		i.metrics.IndexerDuration.WithLabelValues("rede_torrent").Observe(time.Since(start).Seconds())
-		i.metrics.IndexerRequests.WithLabelValues("rede_torrent").Inc()
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
 	}()
 
 	ctx := r.Context()
@@ -40,11 +41,11 @@ func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Reque
 
 	// URL encode query param
 	q = url.QueryEscape(q)
-	url := rede_torrent.URL
+	url := metadata.URL
 	if q != "" {
-		url = fmt.Sprintf("%s%s%s", url, rede_torrent.SearchURL, q)
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
 	} else if page != "" {
-		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, rede_torrent.PagePattern), page)
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
 	}
 
 	fmt.Println("URL:>", url)
@@ -55,7 +56,7 @@ func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Reque
 		if err != nil {
 			fmt.Println(err)
 		}
-		i.metrics.IndexerErrors.WithLabelValues("rede_torrent").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 	defer resp.Close()
@@ -68,7 +69,7 @@ func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Reque
 			fmt.Println(err)
 		}
 
-		i.metrics.IndexerErrors.WithLabelValues("rede_torrent").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 
@@ -78,57 +79,21 @@ func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Reque
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []schema.IndexedTorrent)
-	var errChan = make(chan error)
-	indexedTorrents := []schema.IndexedTorrent{}
-	for _, link := range links {
-		go func(link string) {
-			torrents, err := getTorrentsRedeTorrent(ctx, i, link)
-			if err != nil {
-				fmt.Println(err)
-				errChan <- err
-			}
-			itChan <- torrents
-		}(link)
-	}
-
-	for i := 0; i < len(links); i++ {
-		select {
-		case torrents := <-itChan:
-			indexedTorrents = append(indexedTorrents, torrents...)
-		case err := <-errChan:
-			fmt.Println(err)
-		}
-	}
-
-	for i, it := range indexedTorrents {
-		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
-		qLower := strings.ToLower(q)
-		splitLength := 2
-		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
-	}
-
-	// remove the ones with zero similarity
-	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
-			return it.Similarity > 0
-		})
-	}
-
-	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
-		return int((j.Similarity - i.Similarity) * 1000)
+	// extract each torrent link
+	indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrentsRedeTorrent(ctx, i, link)
 	})
 
-	// send to search index
-	go func() {
-		_ = i.search.IndexTorrents(indexedTorrents)
-	}()
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
+	}
 
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
 	})
 	if err != nil {
 		fmt.Println(err)
@@ -222,7 +187,7 @@ func getTorrentsRedeTorrent(ctx context.Context, i *Indexer, link string) ([]sch
 		}
 	})
 
-	size = stableUniq(size)
+	size = utils.StableUniq(size)
 
 	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
 
@@ -253,7 +218,7 @@ func getTorrentsRedeTorrent(ctx context.Context, i *Indexer, link string) ([]sch
 			}
 
 			ixt := schema.IndexedTorrent{
-				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				Title:         releaseTitle,
 				OriginalTitle: title,
 				Details:       link,
 				Year:          year,

api/starck_filmes.go

@@ -6,12 +6,10 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
-	"slices"
 	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/hbollon/go-edlib"
 
 	"github.com/felipemarinho97/torrent-indexer/magnet"
 	"github.com/felipemarinho97/torrent-indexer/schema"
@@ -20,15 +18,19 @@ import (
 )
 
 var starck_filmes = IndexerMeta{
-	URL:       "https://www.starckfilmes.online/",
-	SearchURL: "?s=",
+	Label:       "starck_filmes",
+	URL:         "https://www.starckfilmes.online/",
+	SearchURL:   "?s=",
+	PagePattern: "page/%s",
 }
 
 func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
+	metadata := starck_filmes
+
 	defer func() {
-		i.metrics.IndexerDuration.WithLabelValues("starck_filmes").Observe(time.Since(start).Seconds())
-		i.metrics.IndexerRequests.WithLabelValues("starck_filmes").Inc()
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
 	}()
 
 	ctx := r.Context()
@@ -38,11 +40,11 @@ func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Requ
 
 	// URL encode query param
 	q = url.QueryEscape(q)
-	url := starck_filmes.URL
+	url := metadata.URL
 	if q != "" {
-		url = fmt.Sprintf("%s%s%s", url, starck_filmes.SearchURL, q)
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
 	} else if page != "" {
-		url = fmt.Sprintf("%spage/%s", url, page)
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
 	}
 
 	fmt.Println("URL:>", url)
@@ -53,7 +55,7 @@ func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Requ
 		if err != nil {
 			fmt.Println(err)
 		}
-		i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 	defer resp.Close()
@@ -66,7 +68,7 @@ func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Requ
 			fmt.Println(err)
 		}
 
-		i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 
@@ -76,57 +78,21 @@ func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Requ
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []schema.IndexedTorrent)
-	var errChan = make(chan error)
-	indexedTorrents := []schema.IndexedTorrent{}
-	for _, link := range links {
-		go func(link string) {
-			torrents, err := getTorrentStarckFilmes(ctx, i, link)
-			if err != nil {
-				fmt.Println(err)
-				errChan <- err
-			}
-			itChan <- torrents
-		}(link)
-	}
-
-	for i := 0; i < len(links); i++ {
-		select {
-		case torrents := <-itChan:
-			indexedTorrents = append(indexedTorrents, torrents...)
-		case err := <-errChan:
-			fmt.Println(err)
-		}
-	}
-
-	for i, it := range indexedTorrents {
-		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
-		qLower := strings.ToLower(q)
-		splitLength := 2
-		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
-	}
-
-	// remove the ones with zero similarity
-	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
-			return it.Similarity > 0
-		})
-	}
-
-	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
-		return int((j.Similarity - i.Similarity) * 1000)
+	// extract each torrent link
+	indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrentStarckFilmes(ctx, i, link)
 	})
 
-	// send to search index
-	go func() {
-		_ = i.search.IndexTorrents(indexedTorrents)
-	}()
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
+	}
 
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
 	})
 	if err != nil {
 		fmt.Println(err)
@@ -184,7 +150,7 @@ func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]sch
 	// TODO: find any link from imdb
 	imdbLink := ""
 
-	size = stableUniq(size)
+	size = utils.StableUniq(size)
 
 	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
 
@@ -228,7 +194,7 @@ func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]sch
 			}
 
 			ixt := schema.IndexedTorrent{
-				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				Title:         releaseTitle,
 				OriginalTitle: title,
 				Details:       link,
 				Year:          year,

api/torrent_dos_filmes.go

@@ -6,12 +6,10 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
-	"slices"
 	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/hbollon/go-edlib"
 
 	"github.com/felipemarinho97/torrent-indexer/magnet"
 	"github.com/felipemarinho97/torrent-indexer/schema"
@@ -20,15 +18,19 @@ import (
 )
 
 var torrent_dos_filmes = IndexerMeta{
-	URL:       "https://torrentdosfilmes.se/",
-	SearchURL: "?s=",
+	Label:       "torrent_dos_filmes",
+	URL:         "https://torrentdosfilmes.se/",
+	SearchURL:   "?s=",
+	PagePattern: "page/%s",
 }
 
 func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
+	metadata := torrent_dos_filmes
+
 	defer func() {
-		i.metrics.IndexerDuration.WithLabelValues("torrent_dos_filmes").Observe(time.Since(start).Seconds())
-		i.metrics.IndexerRequests.WithLabelValues("torrent_dos_filmes").Inc()
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
 	}()
 
 	ctx := r.Context()
@@ -38,11 +40,11 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
 
 	// URL encode query param
 	q = url.QueryEscape(q)
-	url := torrent_dos_filmes.URL
+	url := metadata.URL
 	if q != "" {
-		url = fmt.Sprintf("%s%s%s", url, torrent_dos_filmes.SearchURL, q)
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
 	} else if page != "" {
-		url = fmt.Sprintf("%spage/%s", url, page)
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
 	}
 
 	fmt.Println("URL:>", url)
@@ -53,7 +55,7 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
 		if err != nil {
 			fmt.Println(err)
 		}
-		i.metrics.IndexerErrors.WithLabelValues("torrent_dos_filmes").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 	defer resp.Close()
@@ -66,7 +68,7 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
 			fmt.Println(err)
 		}
 
-		i.metrics.IndexerErrors.WithLabelValues("torrent_dos_filmes").Inc()
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 
@@ -76,57 +78,21 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []schema.IndexedTorrent)
-	var errChan = make(chan error)
-	indexedTorrents := []schema.IndexedTorrent{}
-	for _, link := range links {
-		go func(link string) {
-			torrents, err := getTorrentsTorrentDosFilmes(ctx, i, link)
-			if err != nil {
-				fmt.Println(err)
-				errChan <- err
-			}
-			itChan <- torrents
-		}(link)
-	}
-
-	for i := 0; i < len(links); i++ {
-		select {
-		case torrents := <-itChan:
-			indexedTorrents = append(indexedTorrents, torrents...)
-		case err := <-errChan:
-			fmt.Println(err)
-		}
-	}
-
-	for i, it := range indexedTorrents {
-		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
-		qLower := strings.ToLower(q)
-		splitLength := 2
-		indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
-	}
-
-	// remove the ones with zero similarity
-	if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
-		indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
-			return it.Similarity > 0
-		})
-	}
-
-	// sort by similarity
-	slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
-		return int((j.Similarity - i.Similarity) * 1000)
+	// extract each torrent link
+	indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrentsTorrentDosFilmes(ctx, i, link)
 	})
 
-	// send to search index
-	go func() {
-		_ = i.search.IndexTorrents(indexedTorrents)
-	}()
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
+	}
 
 	w.Header().Set("Content-Type", "application/json")
 	err = json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
 	})
 	if err != nil {
 		fmt.Println(err)
@@ -191,7 +157,7 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
 		}
 	})
 
-	size = stableUniq(size)
+	size = utils.StableUniq(size)
 
 	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
 
@@ -222,7 +188,7 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
 			}
 
 			ixt := schema.IndexedTorrent{
-				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+				Title:         releaseTitle,
 				OriginalTitle: title,
 				Details:       link,
 				Year:          year,