Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f4476024da | |||
|
|
472fedc565 | ||
|
|
6525b5b2b0 | ||
|
|
5bdd2cdd85 | ||
| 0f359ba44e | |||
|
|
bc53fbab1a | ||
| dc3cb3be92 | |||
| 88d6d506bf | |||
| 0a702d1893 | |||
| a6c3d3eed1 | |||
| 6c02f72e13 | |||
| 339db28d5a | |||
| 5034a11a66 | |||
| e994ee109d | |||
| a6977aec0d |
104
README.md
104
README.md
@@ -10,6 +10,9 @@ Visit [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org
|
||||
|
||||
- [comando-torrents](https://comando.la/)
|
||||
- [bludv](https://bludvfilmes.tv/)
|
||||
- [torrent-dos-filmes](https://torrentdosfilmes.se/)
|
||||
- [starck-filmes](https://www.starckfilmes.online/)
|
||||
- [comandohds](https://comandohds.org/)
|
||||
|
||||
## Deploy
|
||||
|
||||
@@ -22,6 +25,21 @@ docker-compose up -d
|
||||
|
||||
The server will be available at [http://localhost:8080/](http://localhost:8080/).
|
||||
|
||||
## Configuration
|
||||
|
||||
You can configure the server using the following environment variables:
|
||||
|
||||
- `PORT`: (optional) The port that the server will listen to. Default: `7006`
|
||||
- `FLARESOLVERR_ADDRESS`: (optional) The address of the FlareSolverr instance. Default: `N/A`
|
||||
- `MEILISEARCH_ADDRESS`: (optional) The address of the MeiliSearch instance. Default: `N/A`
|
||||
- `MEILISEARCH_KEY`: (optional) The API key of the MeiliSearch instance. Default: `N/A`
|
||||
- `REDIS_HOST`: (optional) The address of the Redis instance. Default: `localhost`
|
||||
- `SHORT_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the short-lived cache in duration format. Default: `30m`
|
||||
- This cache is used to cache homepage or search results.
|
||||
- Example: `30m`, `1h`, `1h30m`, `1h30m30s`
|
||||
- `LONG_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the long-lived cache in duration format. Default: `7d`
|
||||
- This cache is used to store the torrent webpages (posts). You can set it to a higher value because the torrent pages are not updated frequently.
|
||||
|
||||
## Integrating with Jackett
|
||||
|
||||
You can integrate this indexer with Jackett by adding a new Torznab custom indexer. Here is an example of how to do it for the `bludv` indexer:
|
||||
@@ -97,7 +115,91 @@ search:
|
||||
# json engine n/a
|
||||
```
|
||||
|
||||
If you have more tips on how to integrate with other torrent API clients like Prowlarr, please open a PR.
|
||||
## Integrating with Prowlarr
|
||||
|
||||
You can integrate this indexer with Prowlarr by adding a custom definition. See [Adding a custom YML definition](https://wiki.servarr.com/prowlarr/indexers#adding-a-custom-yml-definition).
|
||||
|
||||
```yaml
|
||||
---
|
||||
---
|
||||
id: torrent-indexer
|
||||
name: Torrent Indexer
|
||||
description: "Indexing Brazilian Torrent websites into structured data. github.com/felipemarinho97/torrent-indexer"
|
||||
language: pt-BR
|
||||
type: public
|
||||
encoding: UTF-8
|
||||
links:
|
||||
- http://localhost:8080/
|
||||
|
||||
caps:
|
||||
categories:
|
||||
Movies: Movies
|
||||
TV: TV
|
||||
|
||||
modes:
|
||||
search: [q]
|
||||
tv-search: [q, season]
|
||||
movie-search: [q]
|
||||
|
||||
settings:
|
||||
- name: indexer
|
||||
type: select
|
||||
label: Indexer
|
||||
default: bludv
|
||||
options:
|
||||
bludv: BLUDV
|
||||
comando_torrents: Comando Torrents
|
||||
torrent-dos-filmes: Torrent dos Filmes
|
||||
|
||||
search:
|
||||
paths:
|
||||
- path: "/indexers/{{ .Config.indexer }}"
|
||||
response:
|
||||
type: json
|
||||
inputs:
|
||||
filter_results: "true"
|
||||
q: "{{ .Keywords }}"
|
||||
keywordsfilters:
|
||||
- name: tolower
|
||||
- name: re_replace
|
||||
args: ["(?i)(S0)(\\d{1,2})$", "temporada $2"]
|
||||
- name: re_replace
|
||||
args: ["(?i)(S)(\\d{1,3})$", "temporada $2"]
|
||||
|
||||
rows:
|
||||
selector: $.results
|
||||
count:
|
||||
selector: $.count
|
||||
|
||||
fields:
|
||||
download:
|
||||
selector: magnet_link
|
||||
title:
|
||||
selector: title
|
||||
description:
|
||||
selector: original_title
|
||||
details:
|
||||
selector: details
|
||||
infohash:
|
||||
selector: info_hash
|
||||
date:
|
||||
selector: date
|
||||
size:
|
||||
selector: size
|
||||
seeders:
|
||||
selector: seed_count
|
||||
leechers:
|
||||
selector: leech_count
|
||||
imdb:
|
||||
selector: imdb
|
||||
category_is_tv_show:
|
||||
selector: title
|
||||
filters:
|
||||
- name: regexp
|
||||
args: "\\b(S\\d+(?:E\\d+)?)\\b"
|
||||
category:
|
||||
text: "{{ if .Result.category_is_tv_show }}TV{{ else }}Movies{{ end }}"
|
||||
```
|
||||
|
||||
# Warning
|
||||
|
||||
|
||||
38
api/bludv.go
38
api/bludv.go
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -21,7 +20,7 @@ import (
|
||||
)
|
||||
|
||||
var bludv = IndexerMeta{
|
||||
URL: "https://bludvfilmes.tv/",
|
||||
URL: "https://bludv.xyz/",
|
||||
SearchURL: "?s=",
|
||||
}
|
||||
|
||||
@@ -33,13 +32,16 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
}()
|
||||
|
||||
ctx := r.Context()
|
||||
// supported query params: q, season, episode, filter_results
|
||||
// supported query params: q, season, episode, page, filter_results
|
||||
q := r.URL.Query().Get("q")
|
||||
page := r.URL.Query().Get("page")
|
||||
|
||||
// URL encode query param
|
||||
q = url.QueryEscape(q)
|
||||
url := bludv.URL
|
||||
if q != "" {
|
||||
if page != "" {
|
||||
url = fmt.Sprintf("%spage/%s", url, page)
|
||||
} else {
|
||||
url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q)
|
||||
}
|
||||
|
||||
@@ -75,9 +77,9 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
links = append(links, link)
|
||||
})
|
||||
|
||||
var itChan = make(chan []IndexedTorrent)
|
||||
var itChan = make(chan []schema.IndexedTorrent)
|
||||
var errChan = make(chan error)
|
||||
indexedTorrents := []IndexedTorrent{}
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
for _, link := range links {
|
||||
go func(link string) {
|
||||
torrents, err := getTorrentsBluDV(ctx, i, link)
|
||||
@@ -107,16 +109,21 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// remove the ones with zero similarity
|
||||
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
|
||||
return it.Similarity > 0
|
||||
})
|
||||
}
|
||||
|
||||
// sort by similarity
|
||||
slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
|
||||
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
|
||||
return int((j.Similarity - i.Similarity) * 1000)
|
||||
})
|
||||
|
||||
// send to search index
|
||||
go func() {
|
||||
_ = i.search.IndexTorrents(indexedTorrents)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
@@ -127,8 +134,8 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
|
||||
var indexedTorrents []IndexedTorrent
|
||||
func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
|
||||
var indexedTorrents []schema.IndexedTorrent
|
||||
doc, err := getDocument(ctx, i, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -179,16 +186,15 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
|
||||
imdbLink := ""
|
||||
article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Attr("href")
|
||||
re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
|
||||
matches := re.FindStringSubmatch(link)
|
||||
if len(matches) > 0 {
|
||||
imdbLink = matches[0]
|
||||
_imdbLink, err := getIMDBLink(link)
|
||||
if err == nil {
|
||||
imdbLink = _imdbLink
|
||||
}
|
||||
})
|
||||
|
||||
size = stableUniq(size)
|
||||
|
||||
var chanIndexedTorrent = make(chan IndexedTorrent)
|
||||
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
|
||||
|
||||
// for each magnet link, create a new indexed torrent
|
||||
for it, magnetLink := range magnetLinks {
|
||||
@@ -228,7 +234,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
|
||||
mySize = size[it]
|
||||
}
|
||||
|
||||
ixt := IndexedTorrent{
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Details: link,
|
||||
|
||||
@@ -49,14 +49,17 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
|
||||
}()
|
||||
|
||||
ctx := r.Context()
|
||||
// supported query params: q, season, episode
|
||||
// supported query params: q, season, episode, page, filter_results
|
||||
q := r.URL.Query().Get("q")
|
||||
page := r.URL.Query().Get("page")
|
||||
|
||||
// URL encode query param
|
||||
q = url.QueryEscape(q)
|
||||
url := comando.URL
|
||||
if q != "" {
|
||||
url = fmt.Sprintf("%s%s%s", url, comando.SearchURL, q)
|
||||
} else if page != "" {
|
||||
url = fmt.Sprintf("%spage/%s", url, page)
|
||||
}
|
||||
|
||||
fmt.Println("URL:>", url)
|
||||
@@ -90,9 +93,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
|
||||
links = append(links, link)
|
||||
})
|
||||
|
||||
var itChan = make(chan []IndexedTorrent)
|
||||
var itChan = make(chan []schema.IndexedTorrent)
|
||||
var errChan = make(chan error)
|
||||
indexedTorrents := []IndexedTorrent{}
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
for _, link := range links {
|
||||
go func(link string) {
|
||||
torrents, err := getTorrents(ctx, i, link)
|
||||
@@ -122,16 +125,21 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
// remove the ones with zero similarity
|
||||
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
|
||||
return it.Similarity > 0
|
||||
})
|
||||
}
|
||||
|
||||
// sort by similarity
|
||||
slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
|
||||
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
|
||||
return int((j.Similarity - i.Similarity) * 1000)
|
||||
})
|
||||
|
||||
// send to search index
|
||||
go func() {
|
||||
_ = i.search.IndexTorrents(indexedTorrents)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
@@ -142,8 +150,8 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
|
||||
var indexedTorrents []IndexedTorrent
|
||||
func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
|
||||
var indexedTorrents []schema.IndexedTorrent
|
||||
doc, err := getDocument(ctx, i, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -155,19 +163,11 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
|
||||
// div itemprop="datePublished"
|
||||
datePublished := strings.TrimSpace(article.Find("div[itemprop=\"datePublished\"]").Text())
|
||||
// pattern: 10 de setembro de 2021
|
||||
re := regexp.MustCompile(`(\d{2}) de (\w+) de (\d{4})`)
|
||||
matches := re.FindStringSubmatch(datePublished)
|
||||
var date time.Time
|
||||
if len(matches) > 0 {
|
||||
day := matches[1]
|
||||
month := matches[2]
|
||||
year := matches[3]
|
||||
datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
|
||||
date, err = time.Parse("2006-01-02", datePublished)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
date, err := parseLocalizedDate(datePublished)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
magnets := textContent.Find("a[href^=\"magnet\"]")
|
||||
var magnetLinks []string
|
||||
magnets.Each(func(i int, s *goquery.Selection) {
|
||||
@@ -209,16 +209,15 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
|
||||
imdbLink := ""
|
||||
article.Find("a").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Attr("href")
|
||||
re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
|
||||
matches := re.FindStringSubmatch(link)
|
||||
if len(matches) > 0 {
|
||||
imdbLink = matches[0]
|
||||
_imdbLink, err := getIMDBLink(link)
|
||||
if err == nil {
|
||||
imdbLink = _imdbLink
|
||||
}
|
||||
})
|
||||
|
||||
size = stableUniq(size)
|
||||
|
||||
var chanIndexedTorrent = make(chan IndexedTorrent)
|
||||
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
|
||||
|
||||
// for each magnet link, create a new indexed torrent
|
||||
for it, magnetLink := range magnetLinks {
|
||||
@@ -258,7 +257,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
|
||||
mySize = size[it]
|
||||
}
|
||||
|
||||
ixt := IndexedTorrent{
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Details: link,
|
||||
@@ -285,6 +284,40 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
|
||||
return indexedTorrents, nil
|
||||
}
|
||||
|
||||
func getIMDBLink(link string) (string, error) {
|
||||
var imdbLink string
|
||||
re := regexp.MustCompile(`https://www.imdb.com(/[a-z]{2})?/title/(tt\d+)/?`)
|
||||
|
||||
matches := re.FindStringSubmatch(link)
|
||||
if len(matches) > 0 {
|
||||
imdbLink = matches[0]
|
||||
} else {
|
||||
return "", fmt.Errorf("no imdb link found")
|
||||
}
|
||||
return imdbLink, nil
|
||||
}
|
||||
|
||||
func parseLocalizedDate(datePublished string) (time.Time, error) {
|
||||
re := regexp.MustCompile(`(\d{1,2}) de (\w+) de (\d{4})`)
|
||||
matches := re.FindStringSubmatch(datePublished)
|
||||
if len(matches) > 0 {
|
||||
day := matches[1]
|
||||
// append 0 to single digit day
|
||||
if len(day) == 1 {
|
||||
day = fmt.Sprintf("0%s", day)
|
||||
}
|
||||
month := matches[2]
|
||||
year := matches[3]
|
||||
datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
|
||||
date, err := time.Parse("2006-01-02", datePublished)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return date, nil
|
||||
}
|
||||
return time.Time{}, nil
|
||||
}
|
||||
|
||||
func stableUniq(s []string) []string {
|
||||
var uniq []map[string]interface{}
|
||||
m := make(map[string]map[string]interface{})
|
||||
@@ -331,7 +364,7 @@ func findYearFromText(text string, title string) (year string) {
|
||||
year = yearMatch[1]
|
||||
}
|
||||
}
|
||||
return year
|
||||
return strings.TrimSpace(year)
|
||||
}
|
||||
|
||||
func findAudioFromText(text string) []schema.Audio {
|
||||
|
||||
@@ -3,6 +3,7 @@ package handler
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
)
|
||||
@@ -77,3 +78,115 @@ Servidor Via: Torrent
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_parseLocalizedDate(t *testing.T) {
|
||||
type args struct {
|
||||
datePublished string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want time.Time
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "should return date",
|
||||
args: args{
|
||||
datePublished: "12 de outubro de 2022",
|
||||
},
|
||||
want: time.Date(2022, 10, 12, 0, 0, 0, 0, time.UTC),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "should return date single digit",
|
||||
args: args{
|
||||
datePublished: "1 de outubro de 2022",
|
||||
},
|
||||
want: time.Date(2022, 10, 1, 0, 0, 0, 0, time.UTC),
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseLocalizedDate(tt.args.datePublished)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("parseDate() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("parseDate() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_getIMDBLink(t *testing.T) {
|
||||
type args struct {
|
||||
link string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "should return imdb link",
|
||||
args: args{
|
||||
link: "https://www.imdb.com/title/tt1234567",
|
||||
},
|
||||
want: "https://www.imdb.com/title/tt1234567",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "should return imdb link when end with /",
|
||||
args: args{
|
||||
link: "https://www.imdb.com/title/tt1234567/",
|
||||
},
|
||||
want: "https://www.imdb.com/title/tt1234567/",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "should return imdb link when end with /",
|
||||
args: args{
|
||||
link: "https://www.imdb.com/title/tt1234567/",
|
||||
},
|
||||
want: "https://www.imdb.com/title/tt1234567/",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "should return imdb link when it has a language",
|
||||
args: args{
|
||||
link: "https://www.imdb.com/pt/title/tt18722864/",
|
||||
},
|
||||
want: "https://www.imdb.com/pt/title/tt18722864/",
|
||||
},
|
||||
{
|
||||
name: "should return imdb link when it has a language",
|
||||
args: args{
|
||||
link: "https://www.imdb.com/pt/title/tt34608980/",
|
||||
},
|
||||
want: "https://www.imdb.com/pt/title/tt34608980/",
|
||||
},
|
||||
{
|
||||
name: "should return error when link is invalid",
|
||||
args: args{
|
||||
link: "https://www.google.com",
|
||||
},
|
||||
want: "",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := getIMDBLink(tt.args.link)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("getIMDBLink() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("getIMDBLink() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
263
api/comandohds.go
Normal file
263
api/comandohds.go
Normal file
@@ -0,0 +1,263 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
"regexp"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/hbollon/go-edlib"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/magnet"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
)
|
||||
|
||||
var comandohds = IndexerMeta{
|
||||
URL: "https://comandohds.org/",
|
||||
SearchURL: "?s=",
|
||||
}
|
||||
|
||||
var title_re = regexp.MustCompile(`^[(Filme)|(Série)\s]+`)
|
||||
|
||||
func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
i.metrics.IndexerDuration.WithLabelValues("comandohds").Observe(time.Since(start).Seconds())
|
||||
i.metrics.IndexerRequests.WithLabelValues("comandohds").Inc()
|
||||
}()
|
||||
|
||||
ctx := r.Context()
|
||||
// supported query params: q, page, filter_results
|
||||
q := r.URL.Query().Get("q")
|
||||
page := r.URL.Query().Get("page")
|
||||
|
||||
// URL encode query param
|
||||
q = url.QueryEscape(q)
|
||||
url := comandohds.URL
|
||||
if q != "" {
|
||||
url = fmt.Sprintf("%s%s%s", url, comandohds.SearchURL, q)
|
||||
} else if page != "" {
|
||||
url = fmt.Sprintf("%spage/%s", url, page)
|
||||
}
|
||||
|
||||
fmt.Println("URL:>", url)
|
||||
resp, err := i.requester.GetDocument(ctx, url)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
|
||||
return
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
var links []string
|
||||
doc.Find(".post").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Find("div.title > a").Attr("href")
|
||||
links = append(links, link)
|
||||
})
|
||||
|
||||
var itChan = make(chan []schema.IndexedTorrent)
|
||||
var errChan = make(chan error)
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
for _, link := range links {
|
||||
go func(link string) {
|
||||
torrents, err := getTorrentsComandoHDs(ctx, i, link)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
errChan <- err
|
||||
}
|
||||
itChan <- torrents
|
||||
}(link)
|
||||
}
|
||||
|
||||
for i := 0; i < len(links); i++ {
|
||||
select {
|
||||
case torrents := <-itChan:
|
||||
indexedTorrents = append(indexedTorrents, torrents...)
|
||||
case err := <-errChan:
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
for i, it := range indexedTorrents {
|
||||
jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
|
||||
qLower := strings.ToLower(q)
|
||||
splitLength := 2
|
||||
indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
|
||||
}
|
||||
|
||||
// remove the ones with zero similarity
|
||||
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
|
||||
return it.Similarity > 0
|
||||
})
|
||||
}
|
||||
|
||||
// sort by similarity
|
||||
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
|
||||
return int((j.Similarity - i.Similarity) * 1000)
|
||||
})
|
||||
|
||||
// send to search index
|
||||
go func() {
|
||||
_ = i.search.IndexTorrents(indexedTorrents)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
Count: len(indexedTorrents),
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
|
||||
var indexedTorrents []schema.IndexedTorrent
|
||||
doc, err := getDocument(ctx, i, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
article := doc.Find("article")
|
||||
title := title_re.ReplaceAllString(article.Find(".main_title > h1").Text(), "")
|
||||
textContent := article.Find("div.content")
|
||||
date := getPublishedDateTDF(doc)
|
||||
magnets := textContent.Find("a[href^=\"magnet\"]")
|
||||
var magnetLinks []string
|
||||
magnets.Each(func(i int, s *goquery.Selection) {
|
||||
magnetLink, _ := s.Attr("href")
|
||||
magnetLinks = append(magnetLinks, magnetLink)
|
||||
})
|
||||
|
||||
var audio []schema.Audio
|
||||
var year string
|
||||
var size []string
|
||||
article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
|
||||
// pattern:
|
||||
// »INFORMAÇÕES«
|
||||
// Titulo Traduzido: O Guerreiro Banido
|
||||
// Titulo Original: 天龍八部之喬峰傳
|
||||
// <picture />: 5.7
|
||||
// Ano de Lançamento: 2023
|
||||
// Gênero: Ação
|
||||
// Formato: MKV
|
||||
// Qualidade: WEB-DL
|
||||
// Idioma: Português | Inglês
|
||||
// Legenda: Português
|
||||
// Tamanho: – GB
|
||||
// Qualidade Áudio e Vídeo: 10
|
||||
// Duração: 130 Min
|
||||
// Servidor: Torrent
|
||||
text := s.Text()
|
||||
|
||||
audio = append(audio, findAudioFromText(text)...)
|
||||
y := findYearFromText(text, title)
|
||||
if y != "" {
|
||||
year = y
|
||||
}
|
||||
size = append(size, findSizesFromText(text)...)
|
||||
})
|
||||
|
||||
// find any link from imdb
|
||||
imdbLink := ""
|
||||
article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Attr("href")
|
||||
_imdbLink, err := getIMDBLink(link)
|
||||
if err == nil {
|
||||
imdbLink = _imdbLink
|
||||
}
|
||||
})
|
||||
|
||||
size = stableUniq(size)
|
||||
|
||||
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
|
||||
|
||||
// for each magnet link, create a new indexed torrent
|
||||
for it, magnetLink := range magnetLinks {
|
||||
it := it
|
||||
go func(it int, magnetLink string) {
|
||||
magnet, err := magnet.ParseMagnetUri(magnetLink)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
releaseTitle := magnet.DisplayName
|
||||
infoHash := magnet.InfoHash.String()
|
||||
trackers := magnet.Trackers
|
||||
magnetAudio := []schema.Audio{}
|
||||
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
|
||||
magnetAudio = append(magnetAudio, audio...)
|
||||
} else if len(audio) > 1 {
|
||||
// remove portuguese audio, and append to magnetAudio
|
||||
for _, a := range audio {
|
||||
if a != schema.AudioPortuguese {
|
||||
magnetAudio = append(magnetAudio, a)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
magnetAudio = append(magnetAudio, audio...)
|
||||
}
|
||||
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
title := processTitle(title, magnetAudio)
|
||||
|
||||
// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
|
||||
var mySize string
|
||||
if len(size) == len(magnetLinks) {
|
||||
mySize = size[it]
|
||||
}
|
||||
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Details: link,
|
||||
Year: year,
|
||||
IMDB: imdbLink,
|
||||
Audio: magnetAudio,
|
||||
MagnetLink: magnetLink,
|
||||
Date: date,
|
||||
InfoHash: infoHash,
|
||||
Trackers: trackers,
|
||||
LeechCount: peer,
|
||||
SeedCount: seed,
|
||||
Size: mySize,
|
||||
}
|
||||
chanIndexedTorrent <- ixt
|
||||
}(it, magnetLink)
|
||||
}
|
||||
|
||||
for i := 0; i < len(magnetLinks); i++ {
|
||||
it := <-chanIndexedTorrent
|
||||
indexedTorrents = append(indexedTorrents, it)
|
||||
}
|
||||
|
||||
return indexedTorrents, nil
|
||||
}
|
||||
70
api/index.go
70
api/index.go
@@ -9,12 +9,14 @@ import (
|
||||
"github.com/felipemarinho97/torrent-indexer/monitoring"
|
||||
"github.com/felipemarinho97/torrent-indexer/requester"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
meilisearch "github.com/felipemarinho97/torrent-indexer/search"
|
||||
)
|
||||
|
||||
type Indexer struct {
|
||||
redis *cache.Redis
|
||||
metrics *monitoring.Metrics
|
||||
requester *requester.Requster
|
||||
search *meilisearch.SearchIndexer
|
||||
}
|
||||
|
||||
type IndexerMeta struct {
|
||||
@@ -23,32 +25,16 @@ type IndexerMeta struct {
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
Results []IndexedTorrent `json:"results"`
|
||||
Count int `json:"count"`
|
||||
Results []schema.IndexedTorrent `json:"results"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
type IndexedTorrent struct {
|
||||
Title string `json:"title"`
|
||||
OriginalTitle string `json:"original_title"`
|
||||
Details string `json:"details"`
|
||||
Year string `json:"year"`
|
||||
IMDB string `json:"imdb"`
|
||||
Audio []schema.Audio `json:"audio"`
|
||||
MagnetLink string `json:"magnet_link"`
|
||||
Date time.Time `json:"date"`
|
||||
InfoHash string `json:"info_hash"`
|
||||
Trackers []string `json:"trackers"`
|
||||
Size string `json:"size"`
|
||||
LeechCount int `json:"leech_count"`
|
||||
SeedCount int `json:"seed_count"`
|
||||
Similarity float32 `json:"similarity"`
|
||||
}
|
||||
|
||||
func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster) *Indexer {
|
||||
func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster, si *meilisearch.SearchIndexer) *Indexer {
|
||||
return &Indexer{
|
||||
redis: redis,
|
||||
metrics: metrics,
|
||||
requester: req,
|
||||
search: si,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,6 +51,7 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
|
||||
"description": "Indexer for comando torrents",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"page": "page number",
|
||||
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
|
||||
},
|
||||
},
|
||||
@@ -75,9 +62,43 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
|
||||
"description": "Indexer for bludv",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"page": "page number",
|
||||
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
|
||||
}},
|
||||
},
|
||||
"/indexers/torrent-dos-filmes": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"page": "page number",
|
||||
"description": "Indexer for Torrent dos Filmes",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
|
||||
},
|
||||
},
|
||||
},
|
||||
"/indexers/comandohds": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"page": "page number",
|
||||
"description": "Indexer for Comando HDs",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
|
||||
},
|
||||
},
|
||||
},
|
||||
"/indexers/starck-filmes": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"page": "page number",
|
||||
"description": "Indexer for Starck Filmes",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
|
||||
},
|
||||
},
|
||||
},
|
||||
"/indexers/manual": []map[string]interface{}{
|
||||
{
|
||||
"method": "POST",
|
||||
@@ -90,6 +111,15 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
|
||||
"description": "Get all manual torrents",
|
||||
},
|
||||
},
|
||||
"/search": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"description": "Search for cached torrents across all indexers",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -26,7 +26,7 @@ type ManualIndexerRequest struct {
|
||||
func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
var req ManualIndexerRequest
|
||||
indexedTorrents := []IndexedTorrent{}
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
|
||||
// fetch from redis
|
||||
out, err := i.redis.Get(ctx, manualTorrentsRedisKey)
|
||||
@@ -97,7 +97,7 @@ func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
title := processTitle(releaseTitle, magnetAudio)
|
||||
|
||||
ixt := IndexedTorrent{
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Audio: magnetAudio,
|
||||
|
||||
55
api/search.go
Normal file
55
api/search.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
meilisearch "github.com/felipemarinho97/torrent-indexer/search"
|
||||
)
|
||||
|
||||
// MeilisearchHandler handles HTTP requests for Meilisearch integration.
|
||||
type MeilisearchHandler struct {
|
||||
Module *meilisearch.SearchIndexer
|
||||
}
|
||||
|
||||
// NewMeilisearchHandler creates a new instance of MeilisearchHandler.
|
||||
func NewMeilisearchHandler(module *meilisearch.SearchIndexer) *MeilisearchHandler {
|
||||
return &MeilisearchHandler{Module: module}
|
||||
}
|
||||
|
||||
// SearchTorrentHandler handles the searching of torrent items.
|
||||
func (h *MeilisearchHandler) SearchTorrentHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
query := r.URL.Query().Get("q")
|
||||
if query == "" {
|
||||
http.Error(w, "Query parameter 'q' is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
limitStr := r.URL.Query().Get("limit")
|
||||
limit := 10 // Default limit
|
||||
if limitStr != "" {
|
||||
var err error
|
||||
limit, err = strconv.Atoi(limitStr)
|
||||
if err != nil || limit <= 0 {
|
||||
http.Error(w, "Invalid limit parameter", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
results, err := h.Module.SearchTorrent(query, limit)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to search torrents", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(results); err != nil {
|
||||
http.Error(w, "Failed to encode response", http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
253
api/starck_filmes.go
Normal file
253
api/starck_filmes.go
Normal file
@@ -0,0 +1,253 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/hbollon/go-edlib"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/magnet"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
)
|
||||
|
||||
var starck_filmes = IndexerMeta{
|
||||
URL: "https://www.starckfilmes.online/",
|
||||
SearchURL: "?s=",
|
||||
}
|
||||
|
||||
func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
i.metrics.IndexerDuration.WithLabelValues("starck_filmes").Observe(time.Since(start).Seconds())
|
||||
i.metrics.IndexerRequests.WithLabelValues("starck_filmes").Inc()
|
||||
}()
|
||||
|
||||
ctx := r.Context()
|
||||
// supported query params: q, page, filter_results
|
||||
q := r.URL.Query().Get("q")
|
||||
page := r.URL.Query().Get("page")
|
||||
|
||||
// URL encode query param
|
||||
q = url.QueryEscape(q)
|
||||
url := starck_filmes.URL
|
||||
if q != "" {
|
||||
url = fmt.Sprintf("%s%s%s", url, starck_filmes.SearchURL, q)
|
||||
} else if page != "" {
|
||||
url = fmt.Sprintf("%spage/%s", url, page)
|
||||
}
|
||||
|
||||
fmt.Println("URL:>", url)
|
||||
resp, err := i.requester.GetDocument(ctx, url)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
|
||||
return
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
var links []string
|
||||
doc.Find(".item").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Find("div.sub-item > a").Attr("href")
|
||||
links = append(links, link)
|
||||
})
|
||||
|
||||
var itChan = make(chan []schema.IndexedTorrent)
|
||||
var errChan = make(chan error)
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
for _, link := range links {
|
||||
go func(link string) {
|
||||
torrents, err := getTorrentStarckFilmes(ctx, i, link)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
errChan <- err
|
||||
}
|
||||
itChan <- torrents
|
||||
}(link)
|
||||
}
|
||||
|
||||
for i := 0; i < len(links); i++ {
|
||||
select {
|
||||
case torrents := <-itChan:
|
||||
indexedTorrents = append(indexedTorrents, torrents...)
|
||||
case err := <-errChan:
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
for i, it := range indexedTorrents {
|
||||
jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
|
||||
qLower := strings.ToLower(q)
|
||||
splitLength := 2
|
||||
indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
|
||||
}
|
||||
|
||||
// remove the ones with zero similarity
|
||||
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
|
||||
return it.Similarity > 0
|
||||
})
|
||||
}
|
||||
|
||||
// sort by similarity
|
||||
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
|
||||
return int((j.Similarity - i.Similarity) * 1000)
|
||||
})
|
||||
|
||||
// send to search index
|
||||
go func() {
|
||||
_ = i.search.IndexTorrents(indexedTorrents)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
Count: len(indexedTorrents),
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
|
||||
var indexedTorrents []schema.IndexedTorrent
|
||||
doc, err := getDocument(ctx, i, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
post := doc.Find(".post")
|
||||
capa := post.Find(".capa")
|
||||
title := capa.Find(".post-description > h2").Text()
|
||||
post_buttons := post.Find(".post-buttons")
|
||||
magnets := post_buttons.Find("a[href^=\"magnet\"]")
|
||||
var magnetLinks []string
|
||||
magnets.Each(func(i int, s *goquery.Selection) {
|
||||
magnetLink, _ := s.Attr("href")
|
||||
magnetLinks = append(magnetLinks, magnetLink)
|
||||
})
|
||||
|
||||
var audio []schema.Audio
|
||||
var year string
|
||||
var size []string
|
||||
capa.Find(".post-description p").Each(func(i int, s *goquery.Selection) {
|
||||
// pattern:
|
||||
// Nome Original: 28 Weeks Later
|
||||
// Lançamento: 2007
|
||||
// Duração: 1h 40 min
|
||||
// Gênero: Terror, Suspense, Ficção
|
||||
// Formato: MKV
|
||||
// Tamanho: 2.45 GB
|
||||
// Qualidade de Video: 10
|
||||
// Qualidade do Audio: 10
|
||||
// Idioma: Português | Inglês
|
||||
// Legenda: Português, Inglês, Espanhol
|
||||
var text strings.Builder
|
||||
s.Find("span").Each(func (i int, span *goquery.Selection) {
|
||||
text.WriteString(span.Text())
|
||||
text.WriteString(" ")
|
||||
})
|
||||
fmt.Println(text.String())
|
||||
audio = append(audio, findAudioFromText(text.String())...)
|
||||
y := findYearFromText(text.String(), title)
|
||||
if y != "" {
|
||||
year = y
|
||||
}
|
||||
size = append(size, findSizesFromText(text.String())...)
|
||||
})
|
||||
|
||||
// TODO: find any link from imdb
|
||||
imdbLink := ""
|
||||
|
||||
size = stableUniq(size)
|
||||
|
||||
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
|
||||
|
||||
// for each magnet link, create a new indexed torrent
|
||||
for it, magnetLink := range magnetLinks {
|
||||
it := it
|
||||
go func(it int, magnetLink string) {
|
||||
magnet, err := magnet.ParseMagnetUri(magnetLink)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
releaseTitle := magnet.DisplayName
|
||||
infoHash := magnet.InfoHash.String()
|
||||
trackers := magnet.Trackers
|
||||
magnetAudio := []schema.Audio{}
|
||||
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
|
||||
magnetAudio = append(magnetAudio, audio...)
|
||||
} else if len(audio) > 1 {
|
||||
// remove portuguese audio, and append to magnetAudio
|
||||
for _, a := range audio {
|
||||
if a != schema.AudioPortuguese {
|
||||
magnetAudio = append(magnetAudio, a)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
magnetAudio = append(magnetAudio, audio...)
|
||||
}
|
||||
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
title := processTitle(title, magnetAudio)
|
||||
|
||||
// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
|
||||
var mySize string
|
||||
if len(size) == len(magnetLinks) {
|
||||
mySize = size[it]
|
||||
}
|
||||
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Details: link,
|
||||
Year: year,
|
||||
IMDB: imdbLink,
|
||||
Audio: magnetAudio,
|
||||
MagnetLink: magnetLink,
|
||||
Date: time.Now(),
|
||||
InfoHash: infoHash,
|
||||
Trackers: trackers,
|
||||
LeechCount: peer,
|
||||
SeedCount: seed,
|
||||
Size: mySize,
|
||||
}
|
||||
chanIndexedTorrent <- ixt
|
||||
}(it, magnetLink)
|
||||
}
|
||||
|
||||
for i := 0; i < len(magnetLinks); i++ {
|
||||
it := <-chanIndexedTorrent
|
||||
indexedTorrents = append(indexedTorrents, it)
|
||||
}
|
||||
|
||||
return indexedTorrents, nil
|
||||
}
|
||||
273
api/torrent_dos_filmes.go
Normal file
273
api/torrent_dos_filmes.go
Normal file
@@ -0,0 +1,273 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/hbollon/go-edlib"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/magnet"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
)
|
||||
|
||||
var torrent_dos_filmes = IndexerMeta{
|
||||
URL: "https://torrentdosfilmes.se/",
|
||||
SearchURL: "?s=",
|
||||
}
|
||||
|
||||
func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
i.metrics.IndexerDuration.WithLabelValues("torrent_dos_filmes").Observe(time.Since(start).Seconds())
|
||||
i.metrics.IndexerRequests.WithLabelValues("torrent_dos_filmes").Inc()
|
||||
}()
|
||||
|
||||
ctx := r.Context()
|
||||
// supported query params: q, season, episode, page, filter_results
|
||||
q := r.URL.Query().Get("q")
|
||||
page := r.URL.Query().Get("page")
|
||||
|
||||
// URL encode query param
|
||||
q = url.QueryEscape(q)
|
||||
url := torrent_dos_filmes.URL
|
||||
if q != "" {
|
||||
url = fmt.Sprintf("%s%s%s", url, torrent_dos_filmes.SearchURL, q)
|
||||
} else if page != "" {
|
||||
url = fmt.Sprintf("%spage/%s", url, page)
|
||||
}
|
||||
|
||||
fmt.Println("URL:>", url)
|
||||
resp, err := i.requester.GetDocument(ctx, url)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("torrent_dos_filmes").Inc()
|
||||
return
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
i.metrics.IndexerErrors.WithLabelValues("torrent_dos_filmes").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
var links []string
|
||||
doc.Find(".post").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Find("div.title > a").Attr("href")
|
||||
links = append(links, link)
|
||||
})
|
||||
|
||||
var itChan = make(chan []schema.IndexedTorrent)
|
||||
var errChan = make(chan error)
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
for _, link := range links {
|
||||
go func(link string) {
|
||||
torrents, err := getTorrentsTorrentDosFilmes(ctx, i, link)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
errChan <- err
|
||||
}
|
||||
itChan <- torrents
|
||||
}(link)
|
||||
}
|
||||
|
||||
for i := 0; i < len(links); i++ {
|
||||
select {
|
||||
case torrents := <-itChan:
|
||||
indexedTorrents = append(indexedTorrents, torrents...)
|
||||
case err := <-errChan:
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
for i, it := range indexedTorrents {
|
||||
jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
|
||||
qLower := strings.ToLower(q)
|
||||
splitLength := 2
|
||||
indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
|
||||
}
|
||||
|
||||
// remove the ones with zero similarity
|
||||
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
|
||||
return it.Similarity > 0
|
||||
})
|
||||
}
|
||||
|
||||
// sort by similarity
|
||||
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
|
||||
return int((j.Similarity - i.Similarity) * 1000)
|
||||
})
|
||||
|
||||
// send to search index
|
||||
go func() {
|
||||
_ = i.search.IndexTorrents(indexedTorrents)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
Count: len(indexedTorrents),
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
|
||||
var indexedTorrents []schema.IndexedTorrent
|
||||
doc, err := getDocument(ctx, i, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
article := doc.Find("article")
|
||||
title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1)
|
||||
textContent := article.Find("div.content")
|
||||
date := getPublishedDateTDF(doc)
|
||||
magnets := textContent.Find("a[href^=\"magnet\"]")
|
||||
var magnetLinks []string
|
||||
magnets.Each(func(i int, s *goquery.Selection) {
|
||||
magnetLink, _ := s.Attr("href")
|
||||
magnetLinks = append(magnetLinks, magnetLink)
|
||||
})
|
||||
|
||||
var audio []schema.Audio
|
||||
var year string
|
||||
var size []string
|
||||
article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
|
||||
// pattern:
|
||||
// Título Traduzido: Fundação
|
||||
// Título Original: Foundation
|
||||
// IMDb: 7,5
|
||||
// Ano de Lançamento: 2023
|
||||
// Gênero: Ação | Aventura | Ficção
|
||||
// Formato: MKV
|
||||
// Qualidade: WEB-DL
|
||||
// Áudio: Português | Inglês
|
||||
// Idioma: Português | Inglês
|
||||
// Legenda: Português
|
||||
// Tamanho: –
|
||||
// Qualidade de Áudio: 10
|
||||
// Qualidade de Vídeo: 10
|
||||
// Duração: 59 Min.
|
||||
// Servidor: Torrent
|
||||
text := s.Text()
|
||||
|
||||
audio = append(audio, findAudioFromText(text)...)
|
||||
y := findYearFromText(text, title)
|
||||
if y != "" {
|
||||
year = y
|
||||
}
|
||||
size = append(size, findSizesFromText(text)...)
|
||||
})
|
||||
|
||||
// find any link from imdb
|
||||
imdbLink := ""
|
||||
article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Attr("href")
|
||||
_imdbLink, err := getIMDBLink(link)
|
||||
if err == nil {
|
||||
imdbLink = _imdbLink
|
||||
}
|
||||
})
|
||||
|
||||
size = stableUniq(size)
|
||||
|
||||
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
|
||||
|
||||
// for each magnet link, create a new indexed torrent
|
||||
for it, magnetLink := range magnetLinks {
|
||||
it := it
|
||||
go func(it int, magnetLink string) {
|
||||
magnet, err := magnet.ParseMagnetUri(magnetLink)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
releaseTitle := magnet.DisplayName
|
||||
infoHash := magnet.InfoHash.String()
|
||||
trackers := magnet.Trackers
|
||||
magnetAudio := []schema.Audio{}
|
||||
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
|
||||
magnetAudio = append(magnetAudio, audio...)
|
||||
} else if len(audio) > 1 {
|
||||
// remove portuguese audio, and append to magnetAudio
|
||||
for _, a := range audio {
|
||||
if a != schema.AudioPortuguese {
|
||||
magnetAudio = append(magnetAudio, a)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
magnetAudio = append(magnetAudio, audio...)
|
||||
}
|
||||
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
title := processTitle(title, magnetAudio)
|
||||
|
||||
// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
|
||||
var mySize string
|
||||
if len(size) == len(magnetLinks) {
|
||||
mySize = size[it]
|
||||
}
|
||||
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Details: link,
|
||||
Year: year,
|
||||
IMDB: imdbLink,
|
||||
Audio: magnetAudio,
|
||||
MagnetLink: magnetLink,
|
||||
Date: date,
|
||||
InfoHash: infoHash,
|
||||
Trackers: trackers,
|
||||
LeechCount: peer,
|
||||
SeedCount: seed,
|
||||
Size: mySize,
|
||||
}
|
||||
chanIndexedTorrent <- ixt
|
||||
}(it, magnetLink)
|
||||
}
|
||||
|
||||
for i := 0; i < len(magnetLinks); i++ {
|
||||
it := <-chanIndexedTorrent
|
||||
indexedTorrents = append(indexedTorrents, it)
|
||||
}
|
||||
|
||||
return indexedTorrents, nil
|
||||
}
|
||||
|
||||
func getPublishedDateTDF(document *goquery.Document) time.Time {
|
||||
var date time.Time
|
||||
//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">
|
||||
datePublished := strings.TrimSpace(document.Find("meta[property=\"article:published_time\"]").AttrOr("content", ""))
|
||||
|
||||
if datePublished != "" {
|
||||
date, _ = time.Parse(time.RFC3339, datePublished)
|
||||
}
|
||||
|
||||
return date
|
||||
}
|
||||
12
cache/redis.go
vendored
12
cache/redis.go
vendored
@@ -9,13 +9,14 @@ import (
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
var (
|
||||
const (
|
||||
DefaultExpiration = 24 * time.Hour * 7 // 7 days
|
||||
IndexerComandoTorrents = "indexer:comando_torrents"
|
||||
)
|
||||
|
||||
type Redis struct {
|
||||
client *redis.Client
|
||||
client *redis.Client
|
||||
defaultExpiration time.Duration
|
||||
}
|
||||
|
||||
func NewRedis() *Redis {
|
||||
@@ -28,15 +29,20 @@ func NewRedis() *Redis {
|
||||
Addr: fmt.Sprintf("%s:6379", redisHost),
|
||||
Password: "",
|
||||
}),
|
||||
defaultExpiration: DefaultExpiration,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Redis) SetDefaultExpiration(expiration time.Duration) {
|
||||
r.defaultExpiration = expiration
|
||||
}
|
||||
|
||||
func (r *Redis) Get(ctx context.Context, key string) ([]byte, error) {
|
||||
return r.client.Get(ctx, key).Bytes()
|
||||
}
|
||||
|
||||
func (r *Redis) Set(ctx context.Context, key string, value []byte) error {
|
||||
return r.client.Set(ctx, key, value, DefaultExpiration).Err()
|
||||
return r.client.Set(ctx, key, value, r.defaultExpiration).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) SetWithExpiration(ctx context.Context, key string, value []byte, expiration time.Duration) error {
|
||||
|
||||
@@ -11,6 +11,8 @@ services:
|
||||
- indexer
|
||||
environment:
|
||||
- REDIS_HOST=redis
|
||||
- MEILISEARCH_ADDRESS=http://meilisearch:7700
|
||||
- MEILISEARCH_KEY=my-secret-key
|
||||
- FLARESOLVERR_ADDRESS=http://flaresolverr:8191
|
||||
|
||||
redis:
|
||||
@@ -20,5 +22,17 @@ services:
|
||||
networks:
|
||||
- indexer
|
||||
|
||||
# This container is not necessary for the indexer to work,
|
||||
# deploy if you want to use the search feature
|
||||
meilisearch:
|
||||
image: getmeili/meilisearch:latest
|
||||
container_name: meilisearch
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- indexer
|
||||
environment:
|
||||
- MEILI_NO_ANALYTICS=true
|
||||
- MEILI_MASTER_KEY=my-secret-key
|
||||
|
||||
networks:
|
||||
indexer:
|
||||
|
||||
3
go.mod
3
go.mod
@@ -12,7 +12,6 @@ require (
|
||||
github.com/prometheus/client_model v0.6.0 // indirect
|
||||
github.com/prometheus/common v0.50.0 // indirect
|
||||
github.com/prometheus/procfs v0.13.0 // indirect
|
||||
golang.org/x/net v0.22.0 // indirect
|
||||
golang.org/x/sys v0.18.0 // indirect
|
||||
google.golang.org/protobuf v1.33.0 // indirect
|
||||
)
|
||||
@@ -21,4 +20,6 @@ require (
|
||||
github.com/PuerkitoBio/goquery v1.9.1
|
||||
github.com/hbollon/go-edlib v1.6.0
|
||||
github.com/prometheus/client_golang v1.19.0
|
||||
github.com/xhit/go-str2duration/v2 v2.1.0
|
||||
golang.org/x/net v0.22.0
|
||||
)
|
||||
|
||||
2
go.sum
2
go.sum
@@ -28,6 +28,8 @@ github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGK
|
||||
github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
|
||||
github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
|
||||
github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
|
||||
github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
|
||||
github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
|
||||
38
main.go
38
main.go
@@ -8,26 +8,52 @@ import (
|
||||
handler "github.com/felipemarinho97/torrent-indexer/api"
|
||||
"github.com/felipemarinho97/torrent-indexer/cache"
|
||||
"github.com/felipemarinho97/torrent-indexer/monitoring"
|
||||
"github.com/felipemarinho97/torrent-indexer/public"
|
||||
"github.com/felipemarinho97/torrent-indexer/requester"
|
||||
meilisearch "github.com/felipemarinho97/torrent-indexer/search"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
str2duration "github.com/xhit/go-str2duration/v2"
|
||||
)
|
||||
|
||||
func main() {
|
||||
redis := cache.NewRedis()
|
||||
searchIndex := meilisearch.NewSearchIndexer(os.Getenv("MEILISEARCH_ADDRESS"), os.Getenv("MEILISEARCH_KEY"), "torrents")
|
||||
metrics := monitoring.NewMetrics()
|
||||
metrics.Register()
|
||||
|
||||
flaresolverr := requester.NewFlareSolverr(os.Getenv("FLARESOLVERR_ADDRESS"), 60000)
|
||||
req := requester.NewRequester(flaresolverr, redis)
|
||||
indexers := handler.NewIndexers(redis, metrics, req)
|
||||
|
||||
// get shot-lived and long-lived cache expiration from env
|
||||
shortLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("SHORT_LIVED_CACHE_EXPIRATION"))
|
||||
if err == nil {
|
||||
fmt.Printf("Setting short-lived cache expiration to %s\n", shortLivedCacheExpiration)
|
||||
req.SetShortLivedCacheExpiration(shortLivedCacheExpiration)
|
||||
}
|
||||
longLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("LONG_LIVED_CACHE_EXPIRATION"))
|
||||
if err == nil {
|
||||
fmt.Printf("Setting long-lived cache expiration to %s\n", longLivedCacheExpiration)
|
||||
redis.SetDefaultExpiration(longLivedCacheExpiration)
|
||||
} else {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
indexers := handler.NewIndexers(redis, metrics, req, searchIndex)
|
||||
search := handler.NewMeilisearchHandler(searchIndex)
|
||||
|
||||
indexerMux := http.NewServeMux()
|
||||
metricsMux := http.NewServeMux()
|
||||
|
||||
indexerMux.HandleFunc("/", handler.HandlerIndex)
|
||||
indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
|
||||
indexerMux.HandleFunc("/indexers/torrent-dos-filmes", indexers.HandlerTorrentDosFilmesIndexer)
|
||||
indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
|
||||
indexerMux.HandleFunc("/indexers/comandohds", indexers.HandlerComandoHDsIndexer)
|
||||
indexerMux.HandleFunc("/indexers/starck-filmes", indexers.HandlerStarckFilmesIndexer)
|
||||
indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer)
|
||||
indexerMux.HandleFunc("/search", search.SearchTorrentHandler)
|
||||
indexerMux.Handle("/ui/", http.StripPrefix("/ui/", http.FileServer(http.FS(public.UIFiles))))
|
||||
|
||||
metricsMux.Handle("/metrics", promhttp.Handler())
|
||||
|
||||
@@ -37,8 +63,14 @@ func main() {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
fmt.Println("Server listening on :7006")
|
||||
err := http.ListenAndServe(":7006", indexerMux)
|
||||
|
||||
port := os.Getenv("PORT")
|
||||
if port == "" {
|
||||
port = "7006"
|
||||
}
|
||||
|
||||
fmt.Printf("Server listening on :%s\n", port)
|
||||
err = http.ListenAndServe(":"+port, indexerMux)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
6
public/index.go
Normal file
6
public/index.go
Normal file
@@ -0,0 +1,6 @@
|
||||
package public
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed *
|
||||
var UIFiles embed.FS
|
||||
127
public/index.html
Normal file
127
public/index.html
Normal file
@@ -0,0 +1,127 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Torrent Indexer</title>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@heroicons/react/solid@2.0.0/dist/index.umd.js"></script>
|
||||
</head>
|
||||
|
||||
<body class="bg-gray-900 text-white font-sans">
|
||||
<div class="container mx-auto p-6">
|
||||
<!-- Header -->
|
||||
<header class="text-center mb-10">
|
||||
<h1 class="text-4xl font-bold text-blue-400">Torrent Indexer 🇧🇷</h1>
|
||||
<p class="text-gray-400 mt-2">Find torrents with detailed information from torrent-indexer cache</p>
|
||||
</header>
|
||||
|
||||
<!-- Search Bar -->
|
||||
<div class="flex justify-center mb-10">
|
||||
<input id="search-query" type="text" placeholder="Enter search query"
|
||||
class="w-full max-w-lg px-4 py-2 rounded-md border border-gray-600 bg-gray-800 text-white focus:ring focus:ring-blue-500">
|
||||
<button id="search-btn"
|
||||
class="ml-4 px-6 py-2 bg-blue-600 hover:bg-blue-700 rounded-md font-bold text-white">Search</button>
|
||||
</div>
|
||||
|
||||
<!-- Results Section -->
|
||||
<div id="results" class="space-y-6">
|
||||
<!-- Dynamic content will be injected here -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Function to render a single torrent result
|
||||
function renderTorrent(torrent) {
|
||||
return `
|
||||
<div class="p-6 bg-gray-800 rounded-lg shadow-md flex flex-col md:flex-row gap-6">
|
||||
<!-- Torrent Title and Details -->
|
||||
<div class="flex-grow">
|
||||
<h2 class="text-2xl font-bold text-blue-400 flex items-center gap-2">
|
||||
<span>${torrent.title}</span>
|
||||
<span class="text-sm text-gray-400">(${torrent.year})</span>
|
||||
</h2>
|
||||
<p class="text-gray-500 italic mt-1">${torrent.original_title}</p>
|
||||
<div class="mt-4 grid grid-cols-1 sm:grid-cols-2 gap-4">
|
||||
<p><strong>Audio:</strong> ${torrent.audio.join(', ')}</p>
|
||||
<p><strong>Size:</strong> ${torrent.size}</p>
|
||||
<p><strong>Seeds:</strong> ${torrent.seed_count} | <strong>Leeches:</strong> ${torrent.leech_count}</p>
|
||||
<p><strong>Info Hash:</strong> <span class="text-sm break-all text-gray-300">${torrent.info_hash}</span></p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Actions -->
|
||||
<div class="flex flex-col justify-between items-start md:items-end">
|
||||
<div>
|
||||
<a href="${torrent.imdb}" target="_blank"
|
||||
class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M8 16l-4-4m0 0l4-4m-4 4h16" />
|
||||
</svg>
|
||||
View on IMDB
|
||||
</a>
|
||||
<a href="${torrent.details}" target="_blank"
|
||||
class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium mt-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M13 16h-1v-4h-.01M9 20h6a2 2 0 002-2v-5a2 2 0 00-2-2h-3.5a2 2 0 00-1.85 1.19M13 10V6a3 3 0 00-6 0v4" />
|
||||
</svg>
|
||||
View Details
|
||||
</a>
|
||||
</div>
|
||||
<a href="${torrent.magnet_link}" target="_blank"
|
||||
class="px-4 py-2 bg-green-600 hover:bg-green-700 text-white font-bold rounded-md flex items-center gap-2 mt-4">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M9 17v-6m6 6v-6m-6 6l-2-2m8 0l2-2M5 9l7-7 7 7" />
|
||||
</svg>
|
||||
Download Magnet
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
// Handle search
|
||||
async function onSearch() {
|
||||
const query = document.getElementById('search-query').value.trim();
|
||||
if (!query) {
|
||||
alert('Please enter a search query!');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`/search?q=${encodeURIComponent(query)}`);
|
||||
if (!response.ok) {
|
||||
throw new Error('Search failed');
|
||||
}
|
||||
|
||||
const results = await response.json();
|
||||
const resultsContainer = document.getElementById('results');
|
||||
resultsContainer.innerHTML = results.map(renderTorrent).join('');
|
||||
} catch (error) {
|
||||
// add error element
|
||||
document.getElementById('results').innerHTML = `
|
||||
<div class="p-6 bg-red-800 rounded-lg shadow-md text-center">
|
||||
<p class="text-xl font-bold text-red-400">Error fetching search results</p>
|
||||
<p class="text-gray-400 mt-2">Please try again later.</p>
|
||||
</div>
|
||||
`;
|
||||
//alert('Error fetching search results. Please try again.');
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('search-btn').addEventListener('click', onSearch);
|
||||
// on enter press
|
||||
document.getElementById('search-query').addEventListener('keypress', (e) => {
|
||||
if (e.key === 'Enter') {
|
||||
onSearch();
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -6,8 +6,12 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
)
|
||||
|
||||
type FlareSolverr struct {
|
||||
@@ -19,6 +23,10 @@ type FlareSolverr struct {
|
||||
initiated bool
|
||||
}
|
||||
|
||||
var (
|
||||
ErrListSessions = fmt.Errorf("failed to list sessions")
|
||||
)
|
||||
|
||||
func NewFlareSolverr(url string, timeoutMilli int) *FlareSolverr {
|
||||
poolSize := 5
|
||||
httpClient := &http.Client{}
|
||||
@@ -48,6 +56,14 @@ func (f *FlareSolverr) FillSessionPool() error {
|
||||
// Pre-initialize the pool with existing sessions
|
||||
sessions, err := f.ListSessions()
|
||||
if err != nil {
|
||||
// if fail to list sessions, it may not support the sessions.list command
|
||||
// create new dumb sessions to fill the pool
|
||||
if err == ErrListSessions {
|
||||
for len(f.sessionPool) < cap(f.sessionPool) {
|
||||
f.sessionPool <- "dumb-session"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fmt.Println("Failed to list existing FlareSolverr sessions:", err)
|
||||
return err
|
||||
} else {
|
||||
@@ -134,6 +150,9 @@ func (f *FlareSolverr) ListSessions() ([]string, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if sessionsResponse["sessions"] == nil {
|
||||
return nil, ErrListSessions
|
||||
}
|
||||
|
||||
sessions := sessionsResponse["sessions"].([]interface{})
|
||||
var sessionIDs []string
|
||||
@@ -172,7 +191,7 @@ type Response struct {
|
||||
} `json:"solution"`
|
||||
}
|
||||
|
||||
func (f *FlareSolverr) Get(url string) (io.ReadCloser, error) {
|
||||
func (f *FlareSolverr) Get(_url string) (io.ReadCloser, error) {
|
||||
// Check if the FlareSolverr instance was initiated
|
||||
if !f.initiated {
|
||||
return io.NopCloser(bytes.NewReader([]byte(""))), nil
|
||||
@@ -188,7 +207,7 @@ func (f *FlareSolverr) Get(url string) (io.ReadCloser, error) {
|
||||
|
||||
body := map[string]string{
|
||||
"cmd": "request.get",
|
||||
"url": url,
|
||||
"url": _url,
|
||||
"maxTimeout": fmt.Sprintf("%d", f.maxTimeout),
|
||||
"session": session,
|
||||
}
|
||||
@@ -196,6 +215,7 @@ func (f *FlareSolverr) Get(url string) (io.ReadCloser, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -225,6 +245,56 @@ func (f *FlareSolverr) Get(url string) (io.ReadCloser, error) {
|
||||
return nil, fmt.Errorf("under attack")
|
||||
}
|
||||
|
||||
// Return the response body
|
||||
// check if the response is valid HTML
|
||||
if !utils.IsValidHTML(response.Solution.Response) {
|
||||
fmt.Printf("[FlareSolverr] Invalid HTML response from %s\n", _url)
|
||||
response.Solution.Response = ""
|
||||
}
|
||||
|
||||
// If the response body is empty but cookies are present, make a new request
|
||||
if response.Solution.Response == "" && len(response.Solution.Cookies) > 0 {
|
||||
fmt.Printf("[FlareSolverr] Making a new request to %s with cookies\n", _url)
|
||||
// Create a new request with cookies
|
||||
client := &http.Client{}
|
||||
cookieJar, err := cookiejar.New(&cookiejar.Options{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, cookie := range response.Solution.Cookies {
|
||||
cookieJar.SetCookies(&url.URL{Host: cookie.Domain}, []*http.Cookie{
|
||||
{
|
||||
Name: cookie.Name,
|
||||
Value: cookie.Value,
|
||||
Domain: cookie.Domain,
|
||||
Path: cookie.Path,
|
||||
},
|
||||
})
|
||||
}
|
||||
client.Jar = cookieJar
|
||||
|
||||
secondReq, err := http.NewRequest("GET", _url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// use the same user returned by the FlareSolverr
|
||||
secondReq.Header.Set("User-Agent", response.Solution.UserAgent)
|
||||
|
||||
secondResp, err := client.Do(secondReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
respByte := new(bytes.Buffer)
|
||||
_, err = respByte.ReadFrom(secondResp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Return the body of the second request
|
||||
return io.NopCloser(bytes.NewReader(respByte.Bytes())), nil
|
||||
}
|
||||
|
||||
// Return the original response body
|
||||
return io.NopCloser(bytes.NewReader([]byte(response.Solution.Response))), nil
|
||||
}
|
||||
|
||||
@@ -13,20 +13,24 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
shortLivedCacheExpiration = 30 * time.Minute
|
||||
cacheKey = "shortLivedCache"
|
||||
cacheKey = "shortLivedCache"
|
||||
)
|
||||
|
||||
var challangeRegex = regexp.MustCompile(`(?i)(just a moment|cf-chl-bypass|under attack)`)
|
||||
|
||||
type Requster struct {
|
||||
fs *FlareSolverr
|
||||
c *cache.Redis
|
||||
httpClient *http.Client
|
||||
fs *FlareSolverr
|
||||
c *cache.Redis
|
||||
httpClient *http.Client
|
||||
shortLivedCacheExpiration time.Duration
|
||||
}
|
||||
|
||||
func NewRequester(fs *FlareSolverr, c *cache.Redis) *Requster {
|
||||
return &Requster{fs: fs, httpClient: &http.Client{}, c: c}
|
||||
return &Requster{fs: fs, httpClient: &http.Client{}, c: c, shortLivedCacheExpiration: 30 * time.Minute}
|
||||
}
|
||||
|
||||
func (i *Requster) SetShortLivedCacheExpiration(expiration time.Duration) {
|
||||
i.shortLivedCacheExpiration = expiration
|
||||
}
|
||||
|
||||
func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser, error) {
|
||||
@@ -75,7 +79,7 @@ func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser,
|
||||
|
||||
// save response to cache if it's not a challange and body is not empty
|
||||
if !hasChallange(bodyByte) && len(bodyByte) > 0 {
|
||||
err = i.c.SetWithExpiration(ctx, key, bodyByte, shortLivedCacheExpiration)
|
||||
err = i.c.SetWithExpiration(ctx, key, bodyByte, i.shortLivedCacheExpiration)
|
||||
if err != nil {
|
||||
fmt.Printf("failed to save response to cache: %v\n", err)
|
||||
}
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
package schema
|
||||
|
||||
import "strings"
|
||||
|
||||
type Audio string
|
||||
|
||||
const (
|
||||
AudioPortuguese = "Português"
|
||||
AudioPortuguese2 = "Portugues"
|
||||
AudioPortuguese3 = "PT-BR"
|
||||
AudioPortuguese4 = "Dublado"
|
||||
AudioEnglish = "Inglês"
|
||||
AudioEnglish2 = "Ingles"
|
||||
AudioSpanish = "Espanhol"
|
||||
@@ -28,11 +32,18 @@ const (
|
||||
AudioThai = "Tailandês"
|
||||
AudioThai2 = "Tailandes"
|
||||
AudioTurkish = "Turco"
|
||||
AudioHindi = "Hindi"
|
||||
AudioFarsi = "Persa"
|
||||
AudioMalay = "Malaio"
|
||||
AudioDutch = "Holandês"
|
||||
AudioDutch2 = "Holandes"
|
||||
)
|
||||
|
||||
var AudioList = []Audio{
|
||||
AudioPortuguese,
|
||||
AudioPortuguese2,
|
||||
AudioPortuguese3,
|
||||
AudioPortuguese4,
|
||||
AudioEnglish,
|
||||
AudioEnglish2,
|
||||
AudioSpanish,
|
||||
@@ -56,27 +67,36 @@ var AudioList = []Audio{
|
||||
AudioThai,
|
||||
AudioThai2,
|
||||
AudioTurkish,
|
||||
AudioHindi,
|
||||
AudioFarsi,
|
||||
AudioMalay,
|
||||
AudioDutch,
|
||||
AudioDutch2,
|
||||
}
|
||||
|
||||
func (a Audio) String() string {
|
||||
return a.toISO639_2()
|
||||
return a.toTag()
|
||||
}
|
||||
|
||||
func GetAudioFromString(s string) *Audio {
|
||||
for _, a := range AudioList {
|
||||
if string(a) == s {
|
||||
if strings.EqualFold(string(a), s) {
|
||||
return &a
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a Audio) toISO639_2() string {
|
||||
func (a Audio) toTag() string {
|
||||
switch a {
|
||||
case AudioPortuguese:
|
||||
return "por"
|
||||
return "brazilian"
|
||||
case AudioPortuguese2:
|
||||
return "por"
|
||||
return "brazilian"
|
||||
case AudioPortuguese3:
|
||||
return "brazilian"
|
||||
case AudioPortuguese4:
|
||||
return "brazilian"
|
||||
case AudioEnglish:
|
||||
return "eng"
|
||||
case AudioEnglish2:
|
||||
@@ -123,6 +143,16 @@ func (a Audio) toISO639_2() string {
|
||||
return "tha"
|
||||
case AudioTurkish:
|
||||
return "tur"
|
||||
case AudioHindi:
|
||||
return "hin"
|
||||
case AudioFarsi:
|
||||
return "fas"
|
||||
case AudioMalay:
|
||||
return "msa"
|
||||
case AudioDutch:
|
||||
return "nld"
|
||||
case AudioDutch2:
|
||||
return "nld"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
|
||||
20
schema/indexed_torrent.go
Normal file
20
schema/indexed_torrent.go
Normal file
@@ -0,0 +1,20 @@
|
||||
package schema
|
||||
|
||||
import "time"
|
||||
|
||||
type IndexedTorrent struct {
|
||||
Title string `json:"title"`
|
||||
OriginalTitle string `json:"original_title"`
|
||||
Details string `json:"details"`
|
||||
Year string `json:"year"`
|
||||
IMDB string `json:"imdb"`
|
||||
Audio []Audio `json:"audio"`
|
||||
MagnetLink string `json:"magnet_link"`
|
||||
Date time.Time `json:"date"`
|
||||
InfoHash string `json:"info_hash"`
|
||||
Trackers []string `json:"trackers"`
|
||||
Size string `json:"size"`
|
||||
LeechCount int `json:"leech_count"`
|
||||
SeedCount int `json:"seed_count"`
|
||||
Similarity float32 `json:"similarity"`
|
||||
}
|
||||
147
search/meilisearch.go
Normal file
147
search/meilisearch.go
Normal file
@@ -0,0 +1,147 @@
|
||||
package meilisearch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
)
|
||||
|
||||
// SearchIndexer integrates with Meilisearch to index and search torrent items.
|
||||
type SearchIndexer struct {
|
||||
Client *http.Client
|
||||
BaseURL string
|
||||
APIKey string
|
||||
IndexName string
|
||||
}
|
||||
|
||||
// NewSearchIndexer creates a new instance of SearchIndexer.
|
||||
func NewSearchIndexer(baseURL, apiKey, indexName string) *SearchIndexer {
|
||||
return &SearchIndexer{
|
||||
Client: &http.Client{Timeout: 10 * time.Second},
|
||||
BaseURL: baseURL,
|
||||
APIKey: apiKey,
|
||||
IndexName: indexName,
|
||||
}
|
||||
}
|
||||
|
||||
// IndexTorrent indexes a single torrent item in Meilisearch.
|
||||
func (t *SearchIndexer) IndexTorrent(torrent schema.IndexedTorrent) error {
|
||||
url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
|
||||
|
||||
torrentWithKey := struct {
|
||||
Hash string `json:"id"`
|
||||
schema.IndexedTorrent
|
||||
}{
|
||||
Hash: torrent.InfoHash,
|
||||
IndexedTorrent: torrent,
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(torrentWithKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal torrent data: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if t.APIKey != "" {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
|
||||
}
|
||||
|
||||
resp, err := t.Client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *SearchIndexer) IndexTorrents(torrents []schema.IndexedTorrent) error {
|
||||
url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
|
||||
|
||||
torrentsWithKey := make([]struct {
|
||||
Hash string `json:"id"`
|
||||
schema.IndexedTorrent
|
||||
}, 0, len(torrents))
|
||||
for _, torrent := range torrents {
|
||||
torrentWithKey := struct {
|
||||
Hash string `json:"id"`
|
||||
schema.IndexedTorrent
|
||||
}{
|
||||
Hash: torrent.InfoHash,
|
||||
IndexedTorrent: torrent,
|
||||
}
|
||||
torrentsWithKey = append(torrentsWithKey, torrentWithKey)
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(torrentsWithKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal torrent data: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if t.APIKey != "" {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
|
||||
}
|
||||
|
||||
resp, err := t.Client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SearchTorrent searches indexed torrents in Meilisearch based on the query.
|
||||
func (t *SearchIndexer) SearchTorrent(query string, limit int) ([]schema.IndexedTorrent, error) {
|
||||
url := fmt.Sprintf("%s/indexes/%s/search", t.BaseURL, t.IndexName)
|
||||
requestBody := map[string]string{
|
||||
"q": query,
|
||||
}
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal search query: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if t.APIKey != "" {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
|
||||
}
|
||||
|
||||
resp, err := t.Client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to execute request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("search failed: %s", body)
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Hits []schema.IndexedTorrent `json:"hits"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse search response: %w", err)
|
||||
}
|
||||
|
||||
return result.Hits, nil
|
||||
}
|
||||
@@ -1,5 +1,10 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func Filter[A any](arr []A, f func(A) bool) []A {
|
||||
var res []A
|
||||
res = make([]A, 0)
|
||||
@@ -10,3 +15,9 @@ func Filter[A any](arr []A, f func(A) bool) []A {
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func IsValidHTML(input string) bool {
|
||||
r := strings.NewReader(input)
|
||||
_, err := html.Parse(r)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user