Feat/manual indexer (#9)

* new: feat: add manual torrents support

* chg: docs: update documentation

* chg: chore: fix lint ci

---------

Co-authored-by: Felipe Marinho <felipe.marinho@ifood.com.br>

.golangci.yml

@@ -1,3 +1,4 @@
-run:
+issues:
-  skip-files:
+  exclude-files:
     - scrape.go
+    - infohash.go

README.md

@@ -4,9 +4,103 @@ This is a simple torrent indexer that can be used to index torrents from HTML pa
 
 ## Test it
 
-Visit [https://vlambdas.oci.darklyn.online/](https://vlambdas.oci.darklyn.online/) to test it.
+Visit [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org/) to test it.
 
 ## Supported sites
 
 - [comando-torrents](https://comando.la/)
 - [bludv](https://bludvfilmes.tv/)
+
+## Deploy
+
+If you have Docker + docker-compose installed, you can deploy it using the following command:
+
+```bash
+curl -s https://raw.githubusercontent.com/felipemarinho97/torrent-indexer/main/docker-compose.yml > docker-compose.yml
+docker-compose up -d
+```
+
+The server will be available at [http://localhost:8080/](http://localhost:8080/).
+
+## Integrating with Jackett
+
+You can integrate this indexer with Jackett by adding a new Torznab custom indexer. Here is an example of how to do it for the `bludv` indexer:
+
+```yaml
+---
+id: bludv_indexer
+name: BluDV Indexer
+description: "BluDV - Custom indexer on from torrent-indexer"
+language: pt-BR
+type: public
+encoding: UTF-8
+links:
+  - http://localhost:8080/
+
+caps:
+  categorymappings:
+    - { id: Movie, cat: Movies, desc: "Movies" }
+    - { id: TV, cat: TV, desc: "TV" }
+
+  modes:
+    search: [q]
+    tv-search: [q, season, ep]
+    movie-search: [q]
+  allowrawsearch: true
+
+settings: []
+
+search:
+  paths:
+    - path: "indexers/bludv?filter_results=true&q={{ .Keywords }}"
+      response:
+        type: json
+
+  keywordsfilters:
+    - name: tolower
+
+  rows:
+    selector: $.results
+    count:
+      selector: $.count
+
+  fields:
+    _id:
+      selector: title
+    download:
+      selector: magnet_link
+    title:
+      selector: title
+    description:
+      selector: original_title
+    details:
+      selector: details
+    infohash:
+      selector: info_hash
+    date:
+      selector: date
+    size:
+      selector: size
+    seeders:
+      selector: seed_count
+    leechers:
+      selector: leech_count
+    imdb:
+      selector: imdb
+    category_is_tv_show:
+      selector: title
+      filters:
+        - name: regexp
+          args: "\\b(S\\d+(?:E\\d+)?)\\b"
+    category:
+      text: "{{ if .Result.category_is_tv_show }}TV{{ else }}Movie{{ end }}"
+# json engine n/a
+```
+
+If you have more tips on how to integrate with other torrent API clients like Prowlarr, please open a PR.
+
+# Warning
+
+The instance running at [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org/) is my personal instance and it is not guaranteed to be up all the time. Also, for better availability, I recommend deploying your own instance because the Cloudflare protection may block requests from indexed sites if too many requests are made in a short period of time from the same IP.
+
+If I notice that the instance is being used a lot, I may block requests from Jackett to avoid overloading the server without prior notice.

api/index.go

@@ -56,20 +56,35 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
 	err := json.NewEncoder(w).Encode(map[string]interface{}{
 		"time": currentTime,
 		"endpoints": map[string]interface{}{
-			"/indexers/comando_torrents": map[string]interface{}{
+			"/indexers/comando_torrents": []map[string]interface{}{
-				"method":      "GET",
+				{
-				"description": "Indexer for comando torrents",
+					"method":      "GET",
-				"query_params": map[string]string{
+					"description": "Indexer for comando torrents",
-					"q":              "search query",
+					"query_params": map[string]string{
-					"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+						"q":              "search query",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
 				},
 			},
-			"/indexers/bludv": map[string]interface{}{
+			"/indexers/bludv": []map[string]interface{}{
-				"method":      "GET",
+				{
-				"description": "Indexer for bludv",
+					"method":      "GET",
-				"query_params": map[string]string{
+					"description": "Indexer for bludv",
-					"q":              "search query",
+					"query_params": map[string]string{
-					"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+						"q":              "search query",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					}},
+			},
+			"/indexers/manual": []map[string]interface{}{
+				{
+					"method":      "POST",
+					"description": "Add a manual torrent entry to the indexer for 12 hours",
+					"body": map[string]interface{}{
+						"magnetLink": "magnet link",
+					}},
+				{
+					"method":      "GET",
+					"description": "Get all manual torrents",
 				},
 			},
 		},

api/manual.go

@@ -0,0 +1,138 @@
+package handler
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/redis/go-redis/v9"
+)
+
+const manualTorrentsRedisKey = "manual:torrents"
+
+var manualTorrentExpiration = 8 * time.Hour
+
+type ManualIndexerRequest struct {
+	MagnetLink string `json:"magnetLink"`
+}
+
+func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	var req ManualIndexerRequest
+	indexedTorrents := []IndexedTorrent{}
+
+	// fetch from redis
+	out, err := i.redis.Get(ctx, manualTorrentsRedisKey)
+	if err != nil && !errors.Is(err, redis.Nil) {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Println(err)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
+		return
+	} else if errors.Is(err, redis.Nil) {
+		out = bytes.NewBufferString("[]").Bytes()
+	}
+
+	err = json.Unmarshal([]byte(out), &indexedTorrents)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
+		return
+	}
+
+	// check if the request is a POST
+	if r.Method == http.MethodPost {
+		// decode the request body
+		err := json.NewDecoder(r.Body).Decode(&req)
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+			if err != nil {
+				fmt.Println(err)
+			}
+			i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
+			return
+		}
+
+		magnet, err := magnet.ParseMagnetUri(req.MagnetLink)
+		if err != nil {
+			fmt.Println(err)
+		}
+		var audio []schema.Audio
+		releaseTitle := magnet.DisplayName
+		infoHash := magnet.InfoHash.String()
+		trackers := magnet.Trackers
+		magnetAudio := []schema.Audio{}
+		if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
+			magnetAudio = append(magnetAudio, audio...)
+		} else if len(audio) > 1 {
+			// remove portuguese audio, and append to magnetAudio
+			for _, a := range audio {
+				if a != schema.AudioPortuguese {
+					magnetAudio = append(magnetAudio, a)
+				}
+			}
+		} else {
+			magnetAudio = append(magnetAudio, audio...)
+		}
+
+		peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		title := processTitle(releaseTitle, magnetAudio)
+
+		ixt := IndexedTorrent{
+			Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+			OriginalTitle: title,
+			Audio:         magnetAudio,
+			MagnetLink:    req.MagnetLink,
+			InfoHash:      infoHash,
+			Trackers:      trackers,
+			LeechCount:    peer,
+			SeedCount:     seed,
+		}
+
+		// write to redis
+		indexedTorrents = append(indexedTorrents, ixt)
+		out, err := json.Marshal(indexedTorrents)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		err = i.redis.SetWithExpiration(ctx, manualTorrentsRedisKey, out, manualTorrentExpiration)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+			if err != nil {
+				fmt.Println(err)
+			}
+			i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
+			return
+		}
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: indexedTorrents,
+		Count:   len(indexedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}

docker-compose.yml

@@ -1,11 +1,8 @@
-version: '3.7'
+version: '3'
 
 services:
   torrent-indexer:
-    image:
+    image: felipemarinho97/torrent-indexer:latest
-      build:
-        context: .
-        dockerfile: Dockerfile
     container_name: torrent-indexer
     restart: unless-stopped
     ports:

main.go

@@ -21,6 +21,7 @@ func main() {
 	indexerMux.HandleFunc("/", handler.HandlerIndex)
 	indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
 	indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
+	indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer)
 
 	metricsMux.Handle("/metrics", promhttp.Handler())