darklyn/torrent-indexer
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

new: feat: add bludv support

Browse Source
This commit is contained in:
Marinho
2023-10-01 20:27:44 +00:00
parent ae124f4858
commit 67bf0cbf3e
5 changed files with 261 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@@ -2,6 +2,11 @@
This is a simple torrent indexer that can be used to index torrents from HTML pages. It is written in Golang and uses Redis as a cache.
## Test it
Visit [https://vlambdas.oci.darklyn.online/](https://vlambdas.oci.darklyn.online/) to test it.
## Supported sites
- [comando-torrents](https://comando.la/)
- [comando-torrents](https://comando.la/)
- [bludv](https://bludvfilmes.tv/)

215
api/bludv.go Normal file
View File

@@ -0,0 +1,215 @@
package handler
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/felipemarinho97/torrent-indexer/magnet"
"github.com/felipemarinho97/torrent-indexer/schema"
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
)
var bludv = IndexerMeta{
URL: "https://bludvfilmes.tv/",
SearchURL: "?s=",
}
func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
// supported query params: q, season, episode
q := r.URL.Query().Get("q")
// URL encode query param
q = url.QueryEscape(q)
url := bludv.URL
if q != "" {
url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q)
}
fmt.Println("URL:>", url)
resp, err := http.Get(url)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
return
}
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
return
}
var links []string
doc.Find(".post").Each(func(i int, s *goquery.Selection) {
// get link from h2.entry-title > a
link, _ := s.Find("div.title > a").Attr("href")
links = append(links, link)
})
var itChan = make(chan []IndexedTorrent)
var errChan = make(chan error)
indexedTorrents := []IndexedTorrent{}
for _, link := range links {
go func(link string) {
torrents, err := getTorrentsBluDV(ctx, i, link)
if err != nil {
fmt.Println(err)
errChan <- err
}
itChan <- torrents
}(link)
}
for i := 0; i < len(links); i++ {
select {
case torrents := <-itChan:
indexedTorrents = append(indexedTorrents, torrents...)
case err := <-errChan:
fmt.Println(err)
}
}
w.Header().Set("Content-Type", "application/json")
if len(indexedTorrents) == 0 {
w.WriteHeader(http.StatusNotFound)
}
json.NewEncoder(w).Encode(indexedTorrents)
}
func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
var indexedTorrents []IndexedTorrent
doc, err := getDocument(ctx, i, link)
if err != nil {
return nil, err
}
article := doc.Find(".post")
title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1)
textContent := article.Find("div.content")
// div itemprop="datePublished"
datePublished := strings.TrimSpace(article.Find("div[itemprop=\"datePublished\"]").Text())
// pattern: 10 de setembro de 2021
re := regexp.MustCompile(`(\d{2}) de (\w+) de (\d{4})`)
matches := re.FindStringSubmatch(datePublished)
var date time.Time
if len(matches) > 0 {
day := matches[1]
month := matches[2]
year := matches[3]
datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
date, err = time.Parse("2006-01-02", datePublished)
if err != nil {
return nil, err
}
}
magnets := textContent.Find("a[href^=\"magnet\"]")
var magnetLinks []string
magnets.Each(func(i int, s *goquery.Selection) {
magnetLink, _ := s.Attr("href")
magnetLinks = append(magnetLinks, magnetLink)
})
var audio []schema.Audio
var year string
var size []string
article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
// pattern:
// Título Traduzido: Fundação
// Título Original: Foundation
// IMDb: 7,5
// Ano de Lançamento: 2023
// Gênero: Ação | Aventura | Ficção
// Formato: MKV
// Qualidade: WEB-DL
// Áudio: Português | Inglês
// Idioma: Português | Inglês
// Legenda: Português
// Tamanho: –
// Qualidade de Áudio: 10
// Qualidade de Vídeo: 10
// Duração: 59 Min.
// Servidor: Torrent
text := s.Text()
audio = append(audio, findAudioFromText(text)...)
year = findYearFromText(text, title)
size = append(size, findSizesFromText(text)...)
})
size = stableUniq(size)
var chanIndexedTorrent = make(chan IndexedTorrent)
// for each magnet link, create a new indexed torrent
for it, magnetLink := range magnetLinks {
it := it
go func(it int, magnetLink string) {
magnet, err := magnet.ParseMagnetUri(magnetLink)
if err != nil {
fmt.Println(err)
}
releaseTitle := magnet.DisplayName
infoHash := magnet.InfoHash.String()
trackers := magnet.Trackers
magnetAudio := []schema.Audio{}
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
magnetAudio = append(magnetAudio, audio...)
} else if len(audio) > 1 {
// remove portuguese audio, and append to magnetAudio
for _, a := range audio {
if a != schema.AudioPortuguese {
magnetAudio = append(magnetAudio, a)
}
}
} else {
magnetAudio = append(magnetAudio, audio...)
}
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, infoHash, trackers)
if err != nil {
fmt.Println(err)
}
title := processTitle(title, magnetAudio)
// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
var mySize string
if len(size) == len(magnetLinks) {
mySize = size[it]
}
ixt := IndexedTorrent{
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
OriginalTitle: title,
Details: link,
Year: year,
Audio: magnetAudio,
MagnetLink: magnetLink,
Date: date,
InfoHash: infoHash,
Trackers: trackers,
LeechCount: peer,
SeedCount: seed,
Size: mySize,
}
chanIndexedTorrent <- ixt
}(it, magnetLink)
}
for i := 0; i < len(magnetLinks); i++ {
it := <-chanIndexedTorrent
indexedTorrents = append(indexedTorrents, it)
}
return indexedTorrents, nil
}

34
api/comando_torrents.go
View File

@@ -18,10 +18,10 @@ import (
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
)
const (
URL = "https://comando.la/"
queryFilter = "?s="
)
var comando = IndexerMeta{
URL: "https://comando.la/",
SearchURL: "?s=",
}
var replacer = strings.NewReplacer(
"janeiro", "01",
@@ -38,21 +38,6 @@ var replacer = strings.NewReplacer(
"dezembro", "12",
)
type IndexedTorrent struct {
Title string `json:"title"`
OriginalTitle string `json:"original_title"`
Details string `json:"details"`
Year string `json:"year"`
Audio []schema.Audio `json:"audio"`
MagnetLink string `json:"magnet_link"`
Date time.Time `json:"date"`
InfoHash string `json:"info_hash"`
Trackers []string `json:"trackers"`
Size string `json:"size"`
LeechCount int `json:"leech_count"`
SeedCount int `json:"seed_count"`
}
func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
// supported query params: q, season, episode
@@ -60,9 +45,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
// URL encode query param
q = url.QueryEscape(q)
url := URL
url := comando.URL
if q != "" {
url = fmt.Sprintf("%s%s%s", URL, queryFilter, q)
url = fmt.Sprintf("%s%s%s", url, comando.SearchURL, q)
}
fmt.Println("URL:>", url)
@@ -90,7 +75,7 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
var itChan = make(chan []IndexedTorrent)
var errChan = make(chan error)
var indexedTorrents []IndexedTorrent
indexedTorrents := []IndexedTorrent{}
for _, link := range links {
go func(link string) {
torrents, err := getTorrents(ctx, i, link)
@@ -112,6 +97,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
}
w.Header().Set("Content-Type", "application/json")
if len(indexedTorrents) == 0 {
w.WriteHeader(http.StatusNotFound)
}
json.NewEncoder(w).Encode(indexedTorrents)
}
@@ -191,7 +179,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
infoHash := magnet.InfoHash.String()
trackers := magnet.Trackers
magnetAudio := []schema.Audio{}
if strings.Contains(strings.ToLower(releaseTitle), "dual") {
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
magnetAudio = append(magnetAudio, audio...)
} else if len(audio) > 1 {
// remove portuguese audio, and append to magnetAudio

28
api/index.go
View File

@@ -6,12 +6,33 @@ import (
"time"
"github.com/felipemarinho97/torrent-indexer/cache"
"github.com/felipemarinho97/torrent-indexer/schema"
)
type Indexer struct {
redis *cache.Redis
}
type IndexerMeta struct {
URL string
SearchURL string
}
type IndexedTorrent struct {
Title string `json:"title"`
OriginalTitle string `json:"original_title"`
Details string `json:"details"`
Year string `json:"year"`
Audio []schema.Audio `json:"audio"`
MagnetLink string `json:"magnet_link"`
Date time.Time `json:"date"`
InfoHash string `json:"info_hash"`
Trackers []string `json:"trackers"`
Size string `json:"size"`
LeechCount int `json:"leech_count"`
SeedCount int `json:"seed_count"`
}
func NewIndexers(redis *cache.Redis) *Indexer {
return &Indexer{
redis: redis,
@@ -32,6 +53,13 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
"q": "search query",
},
},
"/indexers/bludv": map[string]interface{}{
"method": "GET",
"description": "Indexer for bludv",
"query_params": map[string]string{
"q": "search query",
},
},
},
})
}

1
main.go
View File

@@ -13,6 +13,7 @@ func main() {
http.HandleFunc("/", handler.HandlerIndex)
http.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
http.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
err := http.ListenAndServe(":7006", nil)
if err != nil {