darklyn/torrent-indexer
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Feat/Search support (#25)

Browse Source 
* new: feat: add search support with meilisearch

* new: feat: add search interface

* new: feat: add new audio mappings

* chg: fix: add meilisearch docs

* chg: fix: lint issues

* chg: feat: add br flag

* chg: fix: use the same user agent

* chg: fix: bludv (again)

* chg: fix: lint issue
This commit is contained in:
Felipe Marinho
2024-12-13 11:54:55 -03:00
committed by GitHub
parent 0a702d1893
commit 88d6d506bf
16 changed files with 478 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
api/bludv.go
View File

@@ -40,10 +40,10 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
// URL encode query param
q = url.QueryEscape(q)
url := bludv.URL
if q != "" {
url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q)
} else if page != "" {
if page != "" {
url = fmt.Sprintf("%spage/%s", url, page)
} else {
url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q)
}
fmt.Println("URL:>", url)
@@ -78,9 +78,9 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
links = append(links, link)
})
var itChan = make(chan []IndexedTorrent)
var itChan = make(chan []schema.IndexedTorrent)
var errChan = make(chan error)
indexedTorrents := []IndexedTorrent{}
indexedTorrents := []schema.IndexedTorrent{}
for _, link := range links {
go func(link string) {
torrents, err := getTorrentsBluDV(ctx, i, link)
@@ -110,16 +110,21 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
// remove the ones with zero similarity
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
return it.Similarity > 0
})
}
// sort by similarity
slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
return int((j.Similarity - i.Similarity) * 1000)
})
// send to search index
go func() {
_ = i.search.IndexTorrents(indexedTorrents)
}()
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(Response{
Results: indexedTorrents,
@@ -130,8 +135,8 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
}
}
func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
var indexedTorrents []IndexedTorrent
func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
var indexedTorrents []schema.IndexedTorrent
doc, err := getDocument(ctx, i, link)
if err != nil {
return nil, err
@@ -191,7 +196,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
size = stableUniq(size)
var chanIndexedTorrent = make(chan IndexedTorrent)
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
// for each magnet link, create a new indexed torrent
for it, magnetLink := range magnetLinks {
@@ -231,7 +236,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
mySize = size[it]
}
ixt := IndexedTorrent{
ixt := schema.IndexedTorrent{
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
OriginalTitle: title,
Details: link,

21
api/comando_torrents.go
View File

@@ -93,9 +93,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
links = append(links, link)
})
var itChan = make(chan []IndexedTorrent)
var itChan = make(chan []schema.IndexedTorrent)
var errChan = make(chan error)
indexedTorrents := []IndexedTorrent{}
indexedTorrents := []schema.IndexedTorrent{}
for _, link := range links {
go func(link string) {
torrents, err := getTorrents(ctx, i, link)
@@ -125,16 +125,21 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
// remove the ones with zero similarity
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
return it.Similarity > 0
})
}
// sort by similarity
slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
return int((j.Similarity - i.Similarity) * 1000)
})
// send to search index
go func() {
_ = i.search.IndexTorrents(indexedTorrents)
}()
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(Response{
Results: indexedTorrents,
@@ -145,8 +150,8 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
}
}
func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
var indexedTorrents []IndexedTorrent
func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
var indexedTorrents []schema.IndexedTorrent
doc, err := getDocument(ctx, i, link)
if err != nil {
return nil, err
@@ -221,7 +226,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
size = stableUniq(size)
var chanIndexedTorrent = make(chan IndexedTorrent)
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
// for each magnet link, create a new indexed torrent
for it, magnetLink := range magnetLinks {
@@ -261,7 +266,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
mySize = size[it]
}
ixt := IndexedTorrent{
ixt := schema.IndexedTorrent{
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
OriginalTitle: title,
Details: link,

35
api/index.go
View File

@@ -9,12 +9,14 @@ import (
"github.com/felipemarinho97/torrent-indexer/monitoring"
"github.com/felipemarinho97/torrent-indexer/requester"
"github.com/felipemarinho97/torrent-indexer/schema"
meilisearch "github.com/felipemarinho97/torrent-indexer/search"
)
type Indexer struct {
redis *cache.Redis
metrics *monitoring.Metrics
requester *requester.Requster
search *meilisearch.SearchIndexer
}
type IndexerMeta struct {
@@ -23,32 +25,16 @@ type IndexerMeta struct {
}
type Response struct {
Results []IndexedTorrent `json:"results"`
Count int `json:"count"`
Results []schema.IndexedTorrent `json:"results"`
Count int `json:"count"`
}
type IndexedTorrent struct {
Title string `json:"title"`
OriginalTitle string `json:"original_title"`
Details string `json:"details"`
Year string `json:"year"`
IMDB string `json:"imdb"`
Audio []schema.Audio `json:"audio"`
MagnetLink string `json:"magnet_link"`
Date time.Time `json:"date"`
InfoHash string `json:"info_hash"`
Trackers []string `json:"trackers"`
Size string `json:"size"`
LeechCount int `json:"leech_count"`
SeedCount int `json:"seed_count"`
Similarity float32 `json:"similarity"`
}
func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster) *Indexer {
func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster, si *meilisearch.SearchIndexer) *Indexer {
return &Indexer{
redis: redis,
metrics: metrics,
requester: req,
search: si,
}
}
@@ -103,6 +89,15 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
"description": "Get all manual torrents",
},
},
"/search": []map[string]interface{}{
{
"method": "GET",
"description": "Search for cached torrents across all indexers",
"query_params": map[string]string{
"q": "search query",
},
},
},
},
})
if err != nil {

4
api/manual.go
View File

@@ -26,7 +26,7 @@ type ManualIndexerRequest struct {
func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var req ManualIndexerRequest
indexedTorrents := []IndexedTorrent{}
indexedTorrents := []schema.IndexedTorrent{}
// fetch from redis
out, err := i.redis.Get(ctx, manualTorrentsRedisKey)
@@ -97,7 +97,7 @@ func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
title := processTitle(releaseTitle, magnetAudio)
ixt := IndexedTorrent{
ixt := schema.IndexedTorrent{
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
OriginalTitle: title,
Audio: magnetAudio,

55
api/search.go Normal file
View File

@@ -0,0 +1,55 @@
package handler
import (
"encoding/json"
"net/http"
"strconv"
meilisearch "github.com/felipemarinho97/torrent-indexer/search"
)
// MeilisearchHandler handles HTTP requests for Meilisearch integration.
type MeilisearchHandler struct {
Module *meilisearch.SearchIndexer
}
// NewMeilisearchHandler creates a new instance of MeilisearchHandler.
func NewMeilisearchHandler(module *meilisearch.SearchIndexer) *MeilisearchHandler {
return &MeilisearchHandler{Module: module}
}
// SearchTorrentHandler handles the searching of torrent items.
func (h *MeilisearchHandler) SearchTorrentHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
query := r.URL.Query().Get("q")
if query == "" {
http.Error(w, "Query parameter 'q' is required", http.StatusBadRequest)
return
}
limitStr := r.URL.Query().Get("limit")
limit := 10 // Default limit
if limitStr != "" {
var err error
limit, err = strconv.Atoi(limitStr)
if err != nil || limit <= 0 {
http.Error(w, "Invalid limit parameter", http.StatusBadRequest)
return
}
}
results, err := h.Module.SearchTorrent(query, limit)
if err != nil {
http.Error(w, "Failed to search torrents", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(results); err != nil {
http.Error(w, "Failed to encode response", http.StatusInternalServerError)
}
}

21
api/torrent_dos_filmes.go
View File

@@ -77,9 +77,9 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
links = append(links, link)
})
var itChan = make(chan []IndexedTorrent)
var itChan = make(chan []schema.IndexedTorrent)
var errChan = make(chan error)
indexedTorrents := []IndexedTorrent{}
indexedTorrents := []schema.IndexedTorrent{}
for _, link := range links {
go func(link string) {
torrents, err := getTorrentsTorrentDosFilmes(ctx, i, link)
@@ -109,16 +109,21 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
// remove the ones with zero similarity
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
return it.Similarity > 0
})
}
// sort by similarity
slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
return int((j.Similarity - i.Similarity) * 1000)
})
// send to search index
go func() {
_ = i.search.IndexTorrents(indexedTorrents)
}()
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(Response{
Results: indexedTorrents,
@@ -129,8 +134,8 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
}
}
func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
var indexedTorrents []IndexedTorrent
func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
var indexedTorrents []schema.IndexedTorrent
doc, err := getDocument(ctx, i, link)
if err != nil {
return nil, err
@@ -190,7 +195,7 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
size = stableUniq(size)
var chanIndexedTorrent = make(chan IndexedTorrent)
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
// for each magnet link, create a new indexed torrent
for it, magnetLink := range magnetLinks {
@@ -230,7 +235,7 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
mySize = size[it]
}
ixt := IndexedTorrent{
ixt := schema.IndexedTorrent{
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
OriginalTitle: title,
Details: link,