From 67bf0cbf3eee95fc8b4e55c3d5550c4b5c2a7273 Mon Sep 17 00:00:00 2001 From: Marinho Date: Sun, 1 Oct 2023 20:27:44 +0000 Subject: [PATCH] new: feat: add bludv support --- README.md | 7 +- api/bludv.go | 215 ++++++++++++++++++++++++++++++++++++++++ api/comando_torrents.go | 34 ++----- api/index.go | 28 ++++++ main.go | 1 + 5 files changed, 261 insertions(+), 24 deletions(-) create mode 100644 api/bludv.go diff --git a/README.md b/README.md index 24f7bf1..14b5d84 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,11 @@ This is a simple torrent indexer that can be used to index torrents from HTML pages. It is written in Golang and uses Redis as a cache. +## Test it + +Visit [https://vlambdas.oci.darklyn.online/](https://vlambdas.oci.darklyn.online/) to test it. + ## Supported sites -- [comando-torrents](https://comando.la/) \ No newline at end of file +- [comando-torrents](https://comando.la/) +- [bludv](https://bludvfilmes.tv/) \ No newline at end of file diff --git a/api/bludv.go b/api/bludv.go new file mode 100644 index 0000000..74986d7 --- /dev/null +++ b/api/bludv.go @@ -0,0 +1,215 @@ +package handler + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/felipemarinho97/torrent-indexer/magnet" + "github.com/felipemarinho97/torrent-indexer/schema" + goscrape "github.com/felipemarinho97/torrent-indexer/scrape" +) + +var bludv = IndexerMeta{ + URL: "https://bludvfilmes.tv/", + SearchURL: "?s=", +} + +func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + // supported query params: q, season, episode + q := r.URL.Query().Get("q") + + // URL encode query param + q = url.QueryEscape(q) + url := bludv.URL + if q != "" { + url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q) + } + + fmt.Println("URL:>", url) + resp, err := http.Get(url) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(map[string]string{"error": err.Error()}) + return + } + defer resp.Body.Close() + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(map[string]string{"error": err.Error()}) + return + } + + var links []string + doc.Find(".post").Each(func(i int, s *goquery.Selection) { + // get link from h2.entry-title > a + link, _ := s.Find("div.title > a").Attr("href") + links = append(links, link) + }) + + var itChan = make(chan []IndexedTorrent) + var errChan = make(chan error) + indexedTorrents := []IndexedTorrent{} + for _, link := range links { + go func(link string) { + torrents, err := getTorrentsBluDV(ctx, i, link) + if err != nil { + fmt.Println(err) + errChan <- err + } + itChan <- torrents + }(link) + } + + for i := 0; i < len(links); i++ { + select { + case torrents := <-itChan: + indexedTorrents = append(indexedTorrents, torrents...) + case err := <-errChan: + fmt.Println(err) + } + } + + w.Header().Set("Content-Type", "application/json") + if len(indexedTorrents) == 0 { + w.WriteHeader(http.StatusNotFound) + } + json.NewEncoder(w).Encode(indexedTorrents) +} + +func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) { + var indexedTorrents []IndexedTorrent + doc, err := getDocument(ctx, i, link) + if err != nil { + return nil, err + } + + article := doc.Find(".post") + title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1) + textContent := article.Find("div.content") + // div itemprop="datePublished" + datePublished := strings.TrimSpace(article.Find("div[itemprop=\"datePublished\"]").Text()) + // pattern: 10 de setembro de 2021 + re := regexp.MustCompile(`(\d{2}) de (\w+) de (\d{4})`) + matches := re.FindStringSubmatch(datePublished) + var date time.Time + if len(matches) > 0 { + day := matches[1] + month := matches[2] + year := matches[3] + datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day) + date, err = time.Parse("2006-01-02", datePublished) + if err != nil { + return nil, err + } + } + magnets := textContent.Find("a[href^=\"magnet\"]") + var magnetLinks []string + magnets.Each(func(i int, s *goquery.Selection) { + magnetLink, _ := s.Attr("href") + magnetLinks = append(magnetLinks, magnetLink) + }) + + var audio []schema.Audio + var year string + var size []string + article.Find("div.content p").Each(func(i int, s *goquery.Selection) { + // pattern: + // Título Traduzido: Fundação + // Título Original: Foundation + // IMDb: 7,5 + // Ano de Lançamento: 2023 + // Gênero: Ação | Aventura | Ficção + // Formato: MKV + // Qualidade: WEB-DL + // Áudio: Português | Inglês + // Idioma: Português | Inglês + // Legenda: Português + // Tamanho: – + // Qualidade de Áudio: 10 + // Qualidade de Vídeo: 10 + // Duração: 59 Min. + // Servidor: Torrent + text := s.Text() + + audio = append(audio, findAudioFromText(text)...) + year = findYearFromText(text, title) + size = append(size, findSizesFromText(text)...) + }) + + size = stableUniq(size) + + var chanIndexedTorrent = make(chan IndexedTorrent) + + // for each magnet link, create a new indexed torrent + for it, magnetLink := range magnetLinks { + it := it + go func(it int, magnetLink string) { + magnet, err := magnet.ParseMagnetUri(magnetLink) + if err != nil { + fmt.Println(err) + } + releaseTitle := magnet.DisplayName + infoHash := magnet.InfoHash.String() + trackers := magnet.Trackers + magnetAudio := []schema.Audio{} + if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") { + magnetAudio = append(magnetAudio, audio...) + } else if len(audio) > 1 { + // remove portuguese audio, and append to magnetAudio + for _, a := range audio { + if a != schema.AudioPortuguese { + magnetAudio = append(magnetAudio, a) + } + } + } else { + magnetAudio = append(magnetAudio, audio...) + } + + peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, infoHash, trackers) + if err != nil { + fmt.Println(err) + } + + title := processTitle(title, magnetAudio) + + // if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order + var mySize string + if len(size) == len(magnetLinks) { + mySize = size[it] + } + + ixt := IndexedTorrent{ + Title: appendAudioISO639_2Code(releaseTitle, magnetAudio), + OriginalTitle: title, + Details: link, + Year: year, + Audio: magnetAudio, + MagnetLink: magnetLink, + Date: date, + InfoHash: infoHash, + Trackers: trackers, + LeechCount: peer, + SeedCount: seed, + Size: mySize, + } + chanIndexedTorrent <- ixt + }(it, magnetLink) + } + + for i := 0; i < len(magnetLinks); i++ { + it := <-chanIndexedTorrent + indexedTorrents = append(indexedTorrents, it) + } + + return indexedTorrents, nil +} diff --git a/api/comando_torrents.go b/api/comando_torrents.go index 3d8ef26..08e5923 100644 --- a/api/comando_torrents.go +++ b/api/comando_torrents.go @@ -18,10 +18,10 @@ import ( goscrape "github.com/felipemarinho97/torrent-indexer/scrape" ) -const ( - URL = "https://comando.la/" - queryFilter = "?s=" -) +var comando = IndexerMeta{ + URL: "https://comando.la/", + SearchURL: "?s=", +} var replacer = strings.NewReplacer( "janeiro", "01", @@ -38,21 +38,6 @@ var replacer = strings.NewReplacer( "dezembro", "12", ) -type IndexedTorrent struct { - Title string `json:"title"` - OriginalTitle string `json:"original_title"` - Details string `json:"details"` - Year string `json:"year"` - Audio []schema.Audio `json:"audio"` - MagnetLink string `json:"magnet_link"` - Date time.Time `json:"date"` - InfoHash string `json:"info_hash"` - Trackers []string `json:"trackers"` - Size string `json:"size"` - LeechCount int `json:"leech_count"` - SeedCount int `json:"seed_count"` -} - func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) { ctx := r.Context() // supported query params: q, season, episode @@ -60,9 +45,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) // URL encode query param q = url.QueryEscape(q) - url := URL + url := comando.URL if q != "" { - url = fmt.Sprintf("%s%s%s", URL, queryFilter, q) + url = fmt.Sprintf("%s%s%s", url, comando.SearchURL, q) } fmt.Println("URL:>", url) @@ -90,7 +75,7 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) var itChan = make(chan []IndexedTorrent) var errChan = make(chan error) - var indexedTorrents []IndexedTorrent + indexedTorrents := []IndexedTorrent{} for _, link := range links { go func(link string) { torrents, err := getTorrents(ctx, i, link) @@ -112,6 +97,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) } w.Header().Set("Content-Type", "application/json") + if len(indexedTorrents) == 0 { + w.WriteHeader(http.StatusNotFound) + } json.NewEncoder(w).Encode(indexedTorrents) } @@ -191,7 +179,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent infoHash := magnet.InfoHash.String() trackers := magnet.Trackers magnetAudio := []schema.Audio{} - if strings.Contains(strings.ToLower(releaseTitle), "dual") { + if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") { magnetAudio = append(magnetAudio, audio...) } else if len(audio) > 1 { // remove portuguese audio, and append to magnetAudio diff --git a/api/index.go b/api/index.go index a7920a0..d30702a 100644 --- a/api/index.go +++ b/api/index.go @@ -6,12 +6,33 @@ import ( "time" "github.com/felipemarinho97/torrent-indexer/cache" + "github.com/felipemarinho97/torrent-indexer/schema" ) type Indexer struct { redis *cache.Redis } +type IndexerMeta struct { + URL string + SearchURL string +} + +type IndexedTorrent struct { + Title string `json:"title"` + OriginalTitle string `json:"original_title"` + Details string `json:"details"` + Year string `json:"year"` + Audio []schema.Audio `json:"audio"` + MagnetLink string `json:"magnet_link"` + Date time.Time `json:"date"` + InfoHash string `json:"info_hash"` + Trackers []string `json:"trackers"` + Size string `json:"size"` + LeechCount int `json:"leech_count"` + SeedCount int `json:"seed_count"` +} + func NewIndexers(redis *cache.Redis) *Indexer { return &Indexer{ redis: redis, @@ -32,6 +53,13 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) { "q": "search query", }, }, + "/indexers/bludv": map[string]interface{}{ + "method": "GET", + "description": "Indexer for bludv", + "query_params": map[string]string{ + "q": "search query", + }, + }, }, }) } diff --git a/main.go b/main.go index 7c3662f..3ee9575 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ func main() { http.HandleFunc("/", handler.HandlerIndex) http.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer) + http.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer) err := http.ListenAndServe(":7006", nil) if err != nil {