darklyn/torrent-indexer
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

new: feat: add magnet-metadata-api post processor (#39)

Browse Source 
* new: feat: add magnet-metadata-api post processor

* chg: fix: lint issue

* chg: chore: comment optional containers

* chg: fix: remove redundant check
This commit is contained in:
Felipe Marinho
2025-07-29 12:34:37 -03:00
committed by GitHub
parent e5dea934f1
commit d9141c8df7
14 changed files with 389 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
api/bludv.go
View File

@@ -80,7 +80,7 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrentsBluDV(ctx, i, link)
})
@@ -137,7 +137,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In
// if decoded magnet link is indeed a magnet link, append it
if strings.HasPrefix(magnetLinkDecoded, "magnet:") {
magnetLinks = append(magnetLinks, magnetLinkDecoded)
} else {
} else if !strings.Contains(magnetLinkDecoded, "watch.brplayer") {
fmt.Printf("WARN: link \"%s\" decoding resulted in non-magnet link: %s\n", href, magnetLinkDecoded)
}
})
@@ -211,6 +211,11 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,

7
api/comando_torrents.go
View File

@@ -94,7 +94,7 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrents(ctx, i, link)
})
@@ -208,6 +208,11 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.Indexed
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,

7
api/comandohds.go
View File

@@ -82,7 +82,7 @@ func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Reques
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrentsComandoHDs(ctx, i, link)
})
@@ -192,6 +192,11 @@ func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]sche
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,

40
api/index.go
View File

@@ -6,6 +6,7 @@ import (
"time"
"github.com/felipemarinho97/torrent-indexer/cache"
"github.com/felipemarinho97/torrent-indexer/magnet"
"github.com/felipemarinho97/torrent-indexer/monitoring"
"github.com/felipemarinho97/torrent-indexer/requester"
"github.com/felipemarinho97/torrent-indexer/schema"
@@ -13,11 +14,12 @@ import (
)
type Indexer struct {
redis *cache.Redis
metrics *monitoring.Metrics
requester *requester.Requster
search *meilisearch.SearchIndexer
postProcessors []PostProcessorFunc
redis *cache.Redis
metrics *monitoring.Metrics
requester *requester.Requster
search *meilisearch.SearchIndexer
magnetMetadataAPI *magnet.MetadataClient
postProcessors []PostProcessorFunc
}
type IndexerMeta struct {
@@ -35,19 +37,27 @@ type Response struct {
type PostProcessorFunc func(*Indexer, *http.Request, []schema.IndexedTorrent) []schema.IndexedTorrent
var GlobalPostProcessors = []PostProcessorFunc{
AddSimilarityCheck, // Jaccard similarity
CleanupTitleWebsites, // Remove website names from titles
AppendAudioTags, // Add (brazilian, eng, etc.) audio tags to titles
SendToSearchIndexer, // Send indexed torrents to Meilisearch
AddSimilarityCheck, // Jaccard similarity
FullfilMissingMetadata, // Fill missing size or title metadata
CleanupTitleWebsites, // Remove website names from titles
AppendAudioTags, // Add (brazilian, eng, etc.) audio tags to titles
SendToSearchIndexer, // Send indexed torrents to Meilisearch
}
func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster, si *meilisearch.SearchIndexer) *Indexer {
func NewIndexers(
redis *cache.Redis,
metrics *monitoring.Metrics,
req *requester.Requster,
si *meilisearch.SearchIndexer,
mc *magnet.MetadataClient,
) *Indexer {
return &Indexer{
redis: redis,
metrics: metrics,
requester: req,
search: si,
postProcessors: GlobalPostProcessors,
redis: redis,
metrics: metrics,
requester: req,
search: si,
magnetMetadataAPI: mc,
postProcessors: GlobalPostProcessors,
}
}

39
api/post_processors.go
View File

@@ -35,6 +35,45 @@ func SendToSearchIndexer(i *Indexer, _ *http.Request, torrents []schema.IndexedT
return torrents
}
// FullfilMissingMetadata fills in missing metadata for indexed torrents
func FullfilMissingMetadata(i *Indexer, r *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
if !i.magnetMetadataAPI.IsEnabled() {
return torrents
}
return utils.ParallelFlatMap(torrents, func(it schema.IndexedTorrent) ([]schema.IndexedTorrent, error) {
if it.Size != "" && it.Title != "" && it.OriginalTitle != "" {
return []schema.IndexedTorrent{it}, nil
}
m, err := i.magnetMetadataAPI.FetchMetadata(r.Context(), it.MagnetLink)
if err != nil {
return []schema.IndexedTorrent{it}, nil
}
// convert size in bytes to a human-readable format
it.Size = utils.FormatBytes(m.Size)
// Use name from metadata if available as it is more accurate
if m.Name != "" {
it.Title = m.Name
}
fmt.Printf("hash: %s get -> size: %s\n", m.InfoHash, it.Size)
// If files are present, add them to the indexed torrent
if len(m.Files) > 0 {
it.Files = make([]schema.File, len(m.Files))
for i, file := range m.Files {
it.Files[i] = schema.File{
Path: file.Path,
Size: utils.FormatBytes(file.Size),
}
}
}
return []schema.IndexedTorrent{it}, nil
})
}
func AddSimilarityCheck(i *Indexer, r *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
q := r.URL.Query().Get("q")

7
api/rede_torrent.go
View File

@@ -80,7 +80,7 @@ func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Reque
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrentsRedeTorrent(ctx, i, link)
})
@@ -216,6 +216,11 @@ func getTorrentsRedeTorrent(ctx context.Context, i *Indexer, link string) ([]sch
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,

7
api/starck_filmes.go
View File

@@ -79,7 +79,7 @@ func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Requ
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrentStarckFilmes(ctx, i, link)
})
@@ -192,6 +192,11 @@ func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]sch
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,

9
api/torrent_dos_filmes.go
View File

@@ -21,7 +21,7 @@ var torrent_dos_filmes = IndexerMeta{
Label: "torrent_dos_filmes",
URL: "https://torrentdosfilmes.se/",
SearchURL: "?s=",
PagePattern: "page/%s",
PagePattern: "category/dublado/page/%s",
}
func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.Request) {
@@ -79,7 +79,7 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrentsTorrentDosFilmes(ctx, i, link)
})
@@ -186,6 +186,11 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,