Feat/Add post processors + refactor project (#37)

* chg: feat: clean known patterns from title

* chg: refactor: remove duplicated code, and improve maintainability

* chg: feat: add audio tagging post-processor

* chg: refactor: add generic parallelMap function

* chg: refactor: move more function to common locations

* chg: docs: add func docs

api/index.go

@@ -13,17 +13,18 @@ import (
 )
 
 type Indexer struct {
-	redis     *cache.Redis
-	metrics   *monitoring.Metrics
-	requester *requester.Requster
-	search    *meilisearch.SearchIndexer
+	redis          *cache.Redis
+	metrics        *monitoring.Metrics
+	requester      *requester.Requster
+	search         *meilisearch.SearchIndexer
+	postProcessors []PostProcessorFunc
 }
 
 type IndexerMeta struct {
-	URL       string
-	SearchURL string
-	// pattern for pagination, e.g. "page/%s"
-	PagePattern string
+	Label       string // Label is used for Prometheus metrics and logging. Must be alphanumeric optionally with underscores.
+	URL         string // URL is the base URL of the indexer, e.g. "https://example.com/"
+	SearchURL   string // SearchURL is the base URL for search queries, e.g. "?s="
+	PagePattern string // PagePattern for pagination, e.g. "page/%s"
 }
 
 type Response struct {
@@ -31,12 +32,22 @@ type Response struct {
 	Count   int                     `json:"count"`
 }
 
+type PostProcessorFunc func(*Indexer, *http.Request, []schema.IndexedTorrent) []schema.IndexedTorrent
+
+var GlobalPostProcessors = []PostProcessorFunc{
+	AddSimilarityCheck,   // Jaccard similarity
+	CleanupTitleWebsites, // Remove website names from titles
+	AppendAudioTags,      // Add (brazilian, eng, etc.) audio tags to titles
+	SendToSearchIndexer,  // Send indexed torrents to Meilisearch
+}
+
 func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster, si *meilisearch.SearchIndexer) *Indexer {
 	return &Indexer{
-		redis:     redis,
-		metrics:   metrics,
-		requester: req,
-		search:    si,
+		redis:          redis,
+		metrics:        metrics,
+		requester:      req,
+		search:         si,
+		postProcessors: GlobalPostProcessors,
 	}
 }