3 Commits

Author SHA1 Message Date
a999f065da Feat: Add database stats on search UI (#36) 2025-07-23 18:01:44 -03:00
Douglas Paz
20b054f331 chore: update README, add new indexers (#35) 2025-07-23 13:02:51 -03:00
782f9d6b3f Feat/Rede torrents (#34)
* new: feat: add brand new redetorrent.com indexer

* chg: refactor: create common package

* chg: fix: comandohds formatting

* chg: fix: dual audio detection

* chg: fix: parsing issues

* chg: refactor: remove duplicated code

* chg: refactor: move test funcs to common file
2025-07-22 14:57:05 -03:00
16 changed files with 1054 additions and 337 deletions

View File

@@ -13,6 +13,7 @@ Visit [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org
- [torrent-dos-filmes](https://torrentdosfilmes.se/) - [torrent-dos-filmes](https://torrentdosfilmes.se/)
- [starck-filmes](https://www.starckfilmes.online/) - [starck-filmes](https://www.starckfilmes.online/)
- [comandohds](https://comandohds.org/) - [comandohds](https://comandohds.org/)
- [rede-torrent](https://redetorrent.com/)
## Deploy ## Deploy
@@ -121,7 +122,6 @@ You can integrate this indexer with Prowlarr by adding a custom definition. See
```yaml ```yaml
--- ---
---
id: torrent-indexer id: torrent-indexer
name: Torrent Indexer name: Torrent Indexer
description: "Indexing Brazilian Torrent websites into structured data. github.com/felipemarinho97/torrent-indexer" description: "Indexing Brazilian Torrent websites into structured data. github.com/felipemarinho97/torrent-indexer"
@@ -150,6 +150,9 @@ settings:
bludv: BLUDV bludv: BLUDV
comando_torrents: Comando Torrents comando_torrents: Comando Torrents
torrent-dos-filmes: Torrent dos Filmes torrent-dos-filmes: Torrent dos Filmes
comandohds: Comando HDs
starck-filmes: Starck Filmes
rede_torrent: Rede Torrent
search: search:
paths: paths:

View File

@@ -231,19 +231,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In
releaseTitle := magnet.DisplayName releaseTitle := magnet.DisplayName
infoHash := magnet.InfoHash.String() infoHash := magnet.InfoHash.String()
trackers := magnet.Trackers trackers := magnet.Trackers
magnetAudio := []schema.Audio{} magnetAudio := getAudioFromTitle(releaseTitle, audio)
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
magnetAudio = append(magnetAudio, audio...)
} else if len(audio) > 1 {
// remove portuguese audio, and append to magnetAudio
for _, a := range audio {
if a != schema.AudioPortuguese {
magnetAudio = append(magnetAudio, a)
}
}
} else {
magnetAudio = append(magnetAudio, audio...)
}
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers) peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
if err != nil { if err != nil {

View File

@@ -230,19 +230,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.Indexed
releaseTitle := magnet.DisplayName releaseTitle := magnet.DisplayName
infoHash := magnet.InfoHash.String() infoHash := magnet.InfoHash.String()
trackers := magnet.Trackers trackers := magnet.Trackers
magnetAudio := []schema.Audio{} magnetAudio := getAudioFromTitle(releaseTitle, audio)
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
magnetAudio = append(magnetAudio, audio...)
} else if len(audio) > 1 {
// remove portuguese audio, and append to magnetAudio
for _, a := range audio {
if a != schema.AudioPortuguese {
magnetAudio = append(magnetAudio, a)
}
}
} else {
magnetAudio = append(magnetAudio, audio...)
}
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers) peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
if err != nil { if err != nil {
@@ -284,19 +272,6 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.Indexed
return indexedTorrents, nil return indexedTorrents, nil
} }
func getIMDBLink(link string) (string, error) {
var imdbLink string
re := regexp.MustCompile(`https://www.imdb.com(/[a-z]{2})?/title/(tt\d+)/?`)
matches := re.FindStringSubmatch(link)
if len(matches) > 0 {
imdbLink = matches[0]
} else {
return "", fmt.Errorf("no imdb link found")
}
return imdbLink, nil
}
func parseLocalizedDate(datePublished string) (time.Time, error) { func parseLocalizedDate(datePublished string) (time.Time, error) {
re := regexp.MustCompile(`(\d{1,2}) de (\w+) de (\d{4})`) re := regexp.MustCompile(`(\d{1,2}) de (\w+) de (\d{4})`)
matches := re.FindStringSubmatch(datePublished) matches := re.FindStringSubmatch(datePublished)
@@ -350,56 +325,6 @@ func stableUniq(s []string) []string {
return uniqValues return uniqValues
} }
func findYearFromText(text string, title string) (year string) {
re := regexp.MustCompile(`Lançamento: (.*)`)
yearMatch := re.FindStringSubmatch(text)
if len(yearMatch) > 0 {
year = yearMatch[1]
}
if year == "" {
re = regexp.MustCompile(`\((\d{4})\)`)
yearMatch := re.FindStringSubmatch(title)
if len(yearMatch) > 0 {
year = yearMatch[1]
}
}
return strings.TrimSpace(year)
}
func findAudioFromText(text string) []schema.Audio {
var audio []schema.Audio
re := regexp.MustCompile(`(.udio|Idioma):.?(.*)`)
audioMatch := re.FindStringSubmatch(text)
if len(audioMatch) > 0 {
sep := getSeparator(audioMatch[2])
langs_raw := strings.Split(audioMatch[2], sep)
for _, lang := range langs_raw {
lang = strings.TrimSpace(lang)
a := schema.GetAudioFromString(lang)
if a != nil {
audio = append(audio, *a)
} else {
fmt.Println("unknown language:", lang)
}
}
}
return audio
}
func findSizesFromText(text string) []string {
var sizes []string
// everything that ends with GB or MB, using ',' or '.' as decimal separator
re := regexp.MustCompile(`(\d+[\.,]?\d+) ?(GB|MB)`)
sizesMatch := re.FindAllStringSubmatch(text, -1)
if len(sizesMatch) > 0 {
for _, size := range sizesMatch {
sizes = append(sizes, size[0])
}
}
return sizes
}
func processTitle(title string, a []schema.Audio) string { func processTitle(title string, a []schema.Audio) string {
// remove ' - Donwload' from title // remove ' - Donwload' from title
title = strings.Replace(title, " Download", "", -1) title = strings.Replace(title, " Download", "", -1)
@@ -413,26 +338,6 @@ func processTitle(title string, a []schema.Audio) string {
return title return title
} }
func appendAudioISO639_2Code(title string, a []schema.Audio) string {
if len(a) > 0 {
audio := []string{}
for _, lang := range a {
audio = append(audio, lang.String())
}
title = fmt.Sprintf("%s (%s)", title, strings.Join(audio, ", "))
}
return title
}
func getSeparator(s string) string {
if strings.Contains(s, "|") {
return "|"
} else if strings.Contains(s, ",") {
return ","
}
return " "
}
func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Document, error) { func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Document, error) {
// try to get from redis first // try to get from redis first
docCache, err := i.redis.Get(ctx, link) docCache, err := i.redis.Get(ctx, link)

View File

@@ -4,81 +4,8 @@ import (
"reflect" "reflect"
"testing" "testing"
"time" "time"
"github.com/felipemarinho97/torrent-indexer/schema"
) )
func Test_findAudioFromText(t *testing.T) {
type args struct {
text string
}
tests := []struct {
name string
args args
want []schema.Audio
}{
{
name: "should return audio in portuguese",
args: args{
text: "Áudio: Português",
},
want: []schema.Audio{
schema.AudioPortuguese,
},
},
{
name: "should return audio in portuguese",
args: args{
text: "Idioma: Português",
},
want: []schema.Audio{
schema.AudioPortuguese,
},
},
{
name: "should return audio in portuguese",
args: args{
text: "Audio: Português",
},
want: []schema.Audio{
schema.AudioPortuguese,
},
},
{
name: "should return audio in portuguese",
args: args{
text: `
»INFORMAÇÕES«
Título Traduzido: O Cangaceiro do Futuro
Título Original: O Cangaceiro do Futuro
IMDb: 7,1
Gênero:Comédia
Lançamento: 2022
Qualidade: WEB-DL
Áudio: Português
Legenda: S/L
Formato: MKV
Tamanho: 5.77 GB | 9.60 GB
Duração: 30 Min./Ep.
Qualidade de Áudio: 10
Qualidade de Vídeo: 10
Servidor Via: Torrent
`,
},
want: []schema.Audio{
schema.AudioPortuguese,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := findAudioFromText(tt.args.text); !reflect.DeepEqual(got, tt.want) {
t.Errorf("findAudioFromText() = %v, want %v", got, tt.want)
}
})
}
}
func Test_parseLocalizedDate(t *testing.T) { func Test_parseLocalizedDate(t *testing.T) {
type args struct { type args struct {
datePublished string datePublished string
@@ -119,74 +46,3 @@ func Test_parseLocalizedDate(t *testing.T) {
}) })
} }
} }
func Test_getIMDBLink(t *testing.T) {
type args struct {
link string
}
tests := []struct {
name string
args args
want string
wantErr bool
}{
{
name: "should return imdb link",
args: args{
link: "https://www.imdb.com/title/tt1234567",
},
want: "https://www.imdb.com/title/tt1234567",
wantErr: false,
},
{
name: "should return imdb link when end with /",
args: args{
link: "https://www.imdb.com/title/tt1234567/",
},
want: "https://www.imdb.com/title/tt1234567/",
wantErr: false,
},
{
name: "should return imdb link when end with /",
args: args{
link: "https://www.imdb.com/title/tt1234567/",
},
want: "https://www.imdb.com/title/tt1234567/",
wantErr: false,
},
{
name: "should return imdb link when it has a language",
args: args{
link: "https://www.imdb.com/pt/title/tt18722864/",
},
want: "https://www.imdb.com/pt/title/tt18722864/",
},
{
name: "should return imdb link when it has a language",
args: args{
link: "https://www.imdb.com/pt/title/tt34608980/",
},
want: "https://www.imdb.com/pt/title/tt34608980/",
},
{
name: "should return error when link is invalid",
args: args{
link: "https://www.google.com",
},
want: "",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := getIMDBLink(tt.args.link)
if (err != nil) != tt.wantErr {
t.Errorf("getIMDBLink() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("getIMDBLink() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -6,10 +6,10 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"net/url" "net/url"
"regexp"
"slices" "slices"
"strings" "strings"
"time" "time"
"regexp"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"github.com/hbollon/go-edlib" "github.com/hbollon/go-edlib"
@@ -146,7 +146,7 @@ func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]sche
article := doc.Find("article") article := doc.Find("article")
title := title_re.ReplaceAllString(article.Find(".main_title > h1").Text(), "") title := title_re.ReplaceAllString(article.Find(".main_title > h1").Text(), "")
textContent := article.Find("div.content") textContent := article.Find("div.content")
date := getPublishedDateTDF(doc) date := getPublishedDateFromMeta(doc)
magnets := textContent.Find("a[href^=\"magnet\"]") magnets := textContent.Find("a[href^=\"magnet\"]")
var magnetLinks []string var magnetLinks []string
magnets.Each(func(i int, s *goquery.Selection) { magnets.Each(func(i int, s *goquery.Selection) {
@@ -205,23 +205,15 @@ func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]sche
if err != nil { if err != nil {
fmt.Println(err) fmt.Println(err)
} }
releaseTitle := magnet.DisplayName releaseTitle := strings.TrimSpace(magnet.DisplayName)
infoHash := magnet.InfoHash.String() infoHash := magnet.InfoHash.String()
trackers := magnet.Trackers trackers := magnet.Trackers
magnetAudio := []schema.Audio{} for i, tracker := range trackers {
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") { trackers[i] = strings.TrimSpace(tracker)
magnetAudio = append(magnetAudio, audio...)
} else if len(audio) > 1 {
// remove portuguese audio, and append to magnetAudio
for _, a := range audio {
if a != schema.AudioPortuguese {
magnetAudio = append(magnetAudio, a)
}
}
} else {
magnetAudio = append(magnetAudio, audio...)
} }
magnetAudio := getAudioFromTitle(releaseTitle, audio)
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers) peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
if err != nil { if err != nil {
fmt.Println(err) fmt.Println(err)

187
api/common.go Normal file
View File

@@ -0,0 +1,187 @@
package handler
import (
"fmt"
"regexp"
"slices"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/felipemarinho97/torrent-indexer/schema"
)
func getPublishedDateFromMeta(document *goquery.Document) time.Time {
var date time.Time
//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">
datePublished := strings.TrimSpace(document.Find("meta[property=\"article:published_time\"]").AttrOr("content", ""))
if datePublished != "" {
date, _ = time.Parse(time.RFC3339, datePublished)
}
return date
}
type datePattern struct {
regex *regexp.Regexp
layout string
}
var datePatterns = []datePattern{
{regexp.MustCompile(`\d{4}-\d{2}-\d{2}`), "2006-01-02"},
{regexp.MustCompile(`\d{2}-\d{2}-\d{4}`), "02-01-2006"},
{regexp.MustCompile(`\d{2}/\d{2}/\d{4}`), "02/01/2006"},
}
// getPublishedDateFromRawString extracts the date from a raw string using predefined patterns.
func getPublishedDateFromRawString(dateStr string) time.Time {
for _, p := range datePatterns {
match := p.regex.FindString(dateStr)
if match != "" {
date, err := time.Parse(p.layout, match)
if err == nil {
return date.UTC()
}
}
}
return time.Time{}
}
// getSeparator returns the separator used in the string.
// It checks for common separators like "|", ",", "/", and " e "
func getSeparator(s string) string {
if strings.Contains(s, "|") {
return "|"
} else if strings.Contains(s, ",") {
return ","
} else if strings.Contains(s, "/") {
return "/"
} else if strings.Contains(s, " e ") {
return " e "
}
return " "
}
// findAudioFromText extracts audio languages from a given text.
// It looks for patterns like "Áudio: Português, Inglês" or "Idioma: Português, Inglês"
func findAudioFromText(text string) []schema.Audio {
var audio []schema.Audio
re := regexp.MustCompile(`(.udio|Idioma):.?(.*)`)
audioMatch := re.FindStringSubmatch(text)
if len(audioMatch) > 0 {
sep := getSeparator(audioMatch[2])
langs_raw := strings.Split(audioMatch[2], sep)
for _, lang := range langs_raw {
lang = strings.TrimSpace(lang)
a := schema.GetAudioFromString(lang)
if a != nil {
audio = append(audio, *a)
} else {
fmt.Println("unknown language:", lang)
}
}
}
return audio
}
// findYearFromText extracts the year from a given text.
// It looks for patterns like "Lançamento: 2001" in the title.
func findYearFromText(text string, title string) (year string) {
re := regexp.MustCompile(`Lançamento: (.*)`)
yearMatch := re.FindStringSubmatch(text)
if len(yearMatch) > 0 {
year = yearMatch[1]
}
if year == "" {
re = regexp.MustCompile(`\((\d{4})\)`)
yearMatch := re.FindStringSubmatch(title)
if len(yearMatch) > 0 {
year = yearMatch[1]
}
}
return strings.TrimSpace(year)
}
// findSizesFromText extracts sizes from a given text.
// It looks for patterns like "Tamanho: 1.26 GB" or "Tamanho: 700 MB".
func findSizesFromText(text string) []string {
var sizes []string
// everything that ends with GB or MB, using ',' or '.' as decimal separator
re := regexp.MustCompile(`(\d+[\.,]?\d+) ?(GB|MB)`)
sizesMatch := re.FindAllStringSubmatch(text, -1)
if len(sizesMatch) > 0 {
for _, size := range sizesMatch {
sizes = append(sizes, size[0])
}
}
return sizes
}
// getIMDBLink extracts the IMDB link from a given link.
// It looks for patterns like "https://www.imdb.com/title/tt1234567/".
// Returns an error if no valid IMDB link is found.
func getIMDBLink(link string) (string, error) {
var imdbLink string
re := regexp.MustCompile(`https://www.imdb.com(/[a-z]{2})?/title/(tt\d+)/?`)
matches := re.FindStringSubmatch(link)
if len(matches) > 0 {
imdbLink = matches[0]
} else {
return "", fmt.Errorf("no imdb link found")
}
return imdbLink, nil
}
// appendAudioISO639_2Code appends the audio languages to the title in ISO 639-2 code format.
// It formats the title to include the audio languages in parentheses.
// Example: "Movie Title (eng, por)"
func appendAudioISO639_2Code(title string, a []schema.Audio) string {
if len(a) > 0 {
audio := []string{}
for _, lang := range a {
audio = append(audio, lang.String())
}
title = fmt.Sprintf("%s (%s)", title, strings.Join(audio, ", "))
}
return title
}
// getAudioFromTitle extracts audio languages from the release title.
// It checks for common patterns like "nacional", "dual", or "dublado"
func getAudioFromTitle(releaseTitle string, audioFromContent []schema.Audio) []schema.Audio {
magnetAudio := []schema.Audio{}
isNacional := strings.Contains(strings.ToLower(releaseTitle), "nacional")
if isNacional {
magnetAudio = append(magnetAudio, schema.AudioPortuguese)
}
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
magnetAudio = append(magnetAudio, audioFromContent...)
// if Portuguese audio is not in the audio slice, append it
if !slices.Contains(magnetAudio, schema.AudioPortuguese) {
magnetAudio = append(magnetAudio, schema.AudioPortuguese)
}
} else if len(audioFromContent) > 1 {
// remove portuguese audio, and append to magnetAudio
for _, a := range audioFromContent {
if a != schema.AudioPortuguese {
magnetAudio = append(magnetAudio, a)
}
}
} else {
magnetAudio = append(magnetAudio, audioFromContent...)
}
// order and uniq the audio slice
slices.SortFunc(magnetAudio, func(a, b schema.Audio) int {
return strings.Compare(a.String(), b.String())
})
magnetAudio = slices.Compact(magnetAudio)
return magnetAudio
}

241
api/common_test.go Normal file
View File

@@ -0,0 +1,241 @@
package handler
import (
"reflect"
"testing"
"time"
"github.com/felipemarinho97/torrent-indexer/schema"
)
func Test_getPublishedDateFromRawString(t *testing.T) {
type args struct {
dateStr string
}
tests := []struct {
name string
args args
want time.Time
}{
{
name: "should parse date in format 2025-01-01",
args: args{
dateStr: "2025-01-01",
},
want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
},
{
name: "should parse date in format 01-01-2025",
args: args{
dateStr: "01-01-2025",
},
want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
},
{
name: "should parse date in format 01/01/2025",
args: args{
dateStr: "01/01/2025",
},
want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
},
{
name: "should parse date from starck-filmes link",
args: args{
dateStr: "https://www.starckfilmes.online/catalog/jogos-de-seducao-2025-18-07-2025/",
},
want: time.Date(2025, 7, 18, 0, 0, 0, 0, time.UTC),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := getPublishedDateFromRawString(tt.args.dateStr); !reflect.DeepEqual(got, tt.want) {
t.Errorf("getPublishedDateFromRawString() = %v, want %v", got, tt.want)
}
})
}
}
func Test_findAudioFromText(t *testing.T) {
type args struct {
text string
}
tests := []struct {
name string
args args
want []schema.Audio
}{
{
name: "should return audio in portuguese",
args: args{
text: "Áudio: Português",
},
want: []schema.Audio{
schema.AudioPortuguese,
},
},
{
name: "should return audio in portuguese",
args: args{
text: "Idioma: Português",
},
want: []schema.Audio{
schema.AudioPortuguese,
},
},
{
name: "should return audio in portuguese",
args: args{
text: "Audio: Português",
},
want: []schema.Audio{
schema.AudioPortuguese,
},
},
{
name: "should return audio in portuguese - comando_torrents",
args: args{
text: `
»INFORMAÇÕES«
Título Traduzido: O Cangaceiro do Futuro
Título Original: O Cangaceiro do Futuro
IMDb: 7,1
Gênero:Comédia
Lançamento: 2022
Qualidade: WEB-DL
Áudio: Português
Legenda: S/L
Formato: MKV
Tamanho: 5.77 GB | 9.60 GB
Duração: 30 Min./Ep.
Qualidade de Áudio: 10
Qualidade de Vídeo: 10
Servidor Via: Torrent
`,
},
want: []schema.Audio{
schema.AudioPortuguese,
},
},
{
name: "should return audio in portuguese - rede torrent",
args: args{
text: `
Filme Bicho de Sete Cabeças Torrent
Título Original: Bicho de Sete Cabeças
Lançamento: 2001
Gêneros: Drama / Nacional
Idioma: Português
Qualidade: 720p / BluRay
Duração: 1h 14 Minutos
Formato: Mp4
Vídeo: 10 e Áudio: 10
Legendas: Português
Nota do Imdb: 7.7
Tamanho: 1.26 GB
`,
},
want: []schema.Audio{
schema.AudioPortuguese,
},
},
{
name: "should return audio in portuguese - rede torrent 2",
args: args{
text: `
Filme Branca de Neve e o Caçador Torrent / Assistir Online
Título Original: Snow White and the Huntsman
Lançamento: 2012
Gêneros: Ação / Aventura / Fantasia
Idioma: Português / Inglês
Duração: 126 Minutos
Formato: Mkv / Mp4
Vídeo: 10 e Áudio: 10
Legendas: Sim
Tamanho: 2.69 GB / 1.95 GB / 1.0 GB
`,
},
want: []schema.Audio{
schema.AudioPortuguese,
schema.AudioEnglish,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := findAudioFromText(tt.args.text); !reflect.DeepEqual(got, tt.want) {
t.Errorf("findAudioFromText() = %v, want %v", got, tt.want)
}
})
}
}
func Test_getIMDBLink(t *testing.T) {
type args struct {
link string
}
tests := []struct {
name string
args args
want string
wantErr bool
}{
{
name: "should return imdb link",
args: args{
link: "https://www.imdb.com/title/tt1234567",
},
want: "https://www.imdb.com/title/tt1234567",
wantErr: false,
},
{
name: "should return imdb link when end with /",
args: args{
link: "https://www.imdb.com/title/tt1234567/",
},
want: "https://www.imdb.com/title/tt1234567/",
wantErr: false,
},
{
name: "should return imdb link when end with /",
args: args{
link: "https://www.imdb.com/title/tt1234567/",
},
want: "https://www.imdb.com/title/tt1234567/",
wantErr: false,
},
{
name: "should return imdb link when it has a language",
args: args{
link: "https://www.imdb.com/pt/title/tt18722864/",
},
want: "https://www.imdb.com/pt/title/tt18722864/",
},
{
name: "should return imdb link when it has a language",
args: args{
link: "https://www.imdb.com/pt/title/tt34608980/",
},
want: "https://www.imdb.com/pt/title/tt34608980/",
},
{
name: "should return error when link is invalid",
args: args{
link: "https://www.google.com",
},
want: "",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := getIMDBLink(tt.args.link)
if (err != nil) != tt.wantErr {
t.Errorf("getIMDBLink() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("getIMDBLink() = %v, want %v", got, tt.want)
}
})
}
}

View File

@@ -22,6 +22,8 @@ type Indexer struct {
type IndexerMeta struct { type IndexerMeta struct {
URL string URL string
SearchURL string SearchURL string
// pattern for pagination, e.g. "page/%s"
PagePattern string
} }
type Response struct { type Response struct {
@@ -99,6 +101,17 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
}, },
}, },
}, },
"/indexers/rede_torrent": []map[string]interface{}{
{
"method": "GET",
"description": "Indexer for rede torrent",
"query_params": map[string]string{
"q": "search query",
"page": "page number",
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
},
},
},
"/indexers/manual": []map[string]interface{}{ "/indexers/manual": []map[string]interface{}{
{ {
"method": "POST", "method": "POST",
@@ -120,6 +133,12 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
}, },
}, },
}, },
"/ui/": []map[string]interface{}{
{
"method": "GET",
"description": "Show the unified search UI (only work if Meilisearch is enabled)",
},
},
}, },
}) })
if err != nil { if err != nil {

View File

@@ -6,7 +6,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"net/http" "net/http"
"strings"
"time" "time"
"github.com/felipemarinho97/torrent-indexer/magnet" "github.com/felipemarinho97/torrent-indexer/magnet"
@@ -76,19 +75,7 @@ func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
releaseTitle := magnet.DisplayName releaseTitle := magnet.DisplayName
infoHash := magnet.InfoHash.String() infoHash := magnet.InfoHash.String()
trackers := magnet.Trackers trackers := magnet.Trackers
magnetAudio := []schema.Audio{} magnetAudio := getAudioFromTitle(releaseTitle, audio)
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
magnetAudio = append(magnetAudio, audio...)
} else if len(audio) > 1 {
// remove portuguese audio, and append to magnetAudio
for _, a := range audio {
if a != schema.AudioPortuguese {
magnetAudio = append(magnetAudio, a)
}
}
} else {
magnetAudio = append(magnetAudio, audio...)
}
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers) peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
if err != nil { if err != nil {

280
api/rede_torrent.go Normal file
View File

@@ -0,0 +1,280 @@
package handler
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"regexp"
"slices"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/hbollon/go-edlib"
"github.com/felipemarinho97/torrent-indexer/magnet"
"github.com/felipemarinho97/torrent-indexer/schema"
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
"github.com/felipemarinho97/torrent-indexer/utils"
)
var rede_torrent = IndexerMeta{
URL: "https://redetorrent.com/",
SearchURL: "index.php?s=",
PagePattern: "%s",
}
func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Request) {
start := time.Now()
defer func() {
i.metrics.IndexerDuration.WithLabelValues("rede_torrent").Observe(time.Since(start).Seconds())
i.metrics.IndexerRequests.WithLabelValues("rede_torrent").Inc()
}()
ctx := r.Context()
// supported query params: q, season, episode, page, filter_results
q := r.URL.Query().Get("q")
page := r.URL.Query().Get("page")
// URL encode query param
q = url.QueryEscape(q)
url := rede_torrent.URL
if q != "" {
url = fmt.Sprintf("%s%s%s", url, rede_torrent.SearchURL, q)
} else if page != "" {
url = fmt.Sprintf(fmt.Sprintf("%s%s", url, rede_torrent.PagePattern), page)
}
fmt.Println("URL:>", url)
resp, err := i.requester.GetDocument(ctx, url)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
if err != nil {
fmt.Println(err)
}
i.metrics.IndexerErrors.WithLabelValues("rede_torrent").Inc()
return
}
defer resp.Close()
doc, err := goquery.NewDocumentFromReader(resp)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
if err != nil {
fmt.Println(err)
}
i.metrics.IndexerErrors.WithLabelValues("rede_torrent").Inc()
return
}
var links []string
doc.Find(".capa_lista").Each(func(i int, s *goquery.Selection) {
link, _ := s.Find("a").Attr("href")
links = append(links, link)
})
var itChan = make(chan []schema.IndexedTorrent)
var errChan = make(chan error)
indexedTorrents := []schema.IndexedTorrent{}
for _, link := range links {
go func(link string) {
torrents, err := getTorrentsRedeTorrent(ctx, i, link)
if err != nil {
fmt.Println(err)
errChan <- err
}
itChan <- torrents
}(link)
}
for i := 0; i < len(links); i++ {
select {
case torrents := <-itChan:
indexedTorrents = append(indexedTorrents, torrents...)
case err := <-errChan:
fmt.Println(err)
}
}
for i, it := range indexedTorrents {
jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
qLower := strings.ToLower(q)
splitLength := 2
indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
}
// remove the ones with zero similarity
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
return it.Similarity > 0
})
}
// sort by similarity
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
return int((j.Similarity - i.Similarity) * 1000)
})
// send to search index
go func() {
_ = i.search.IndexTorrents(indexedTorrents)
}()
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(Response{
Results: indexedTorrents,
Count: len(indexedTorrents),
})
if err != nil {
fmt.Println(err)
}
}
func getTorrentsRedeTorrent(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
var indexedTorrents []schema.IndexedTorrent
doc, err := getDocument(ctx, i, link)
if err != nil {
return nil, err
}
article := doc.Find(".conteudo")
// title pattern: "Something - optional balbla (dddd) some shit" - extract "Something" and "dddd"
titleRe := regexp.MustCompile(`^(.*?)(?: - (.*?))? \((\d{4})\)`)
titleP := titleRe.FindStringSubmatch(article.Find("h1").Text())
if len(titleP) < 3 {
return nil, fmt.Errorf("could not extract title from %s", link)
}
title := strings.TrimSpace(titleP[1])
year := strings.TrimSpace(titleP[3])
textContent := article.Find(".apenas_itemprop")
date := getPublishedDateFromMeta(doc)
magnets := textContent.Find("a[href^=\"magnet\"]")
var magnetLinks []string
magnets.Each(func(i int, s *goquery.Selection) {
magnetLink, _ := s.Attr("href")
magnetLinks = append(magnetLinks, magnetLink)
})
var audio []schema.Audio
var size []string
article.Find("div#informacoes > p").Each(func(i int, s *goquery.Selection) {
// pattern:
// Filme Bicho de Sete Cabeças Torrent
// Título Original: Bicho de Sete Cabeças
// Lançamento: 2001
// Gêneros: Drama / Nacional
// Idioma: Português
// Qualidade: 720p / BluRay
// Duração: 1h 14 Minutos
// Formato: Mp4
// Vídeo: 10 e Áudio: 10
// Legendas: Português
// Nota do Imdb: 7.7
// Tamanho: 1.26 GB
// we need to manualy parse because the text is not well formatted
htmlContent, err := s.Html()
if err != nil {
fmt.Println(err)
return
}
// remove any \n and \t characters
htmlContent = strings.ReplaceAll(htmlContent, "\n", "")
htmlContent = strings.ReplaceAll(htmlContent, "\t", "")
// split by <br> tags and render each line
brRe := regexp.MustCompile(`<br\s*\/?>`)
htmlContent = brRe.ReplaceAllString(htmlContent, "<br>")
lines := strings.Split(htmlContent, "<br>")
var text strings.Builder
for _, line := range lines {
// remove any HTML tags
re := regexp.MustCompile(`<[^>]*>`)
line = re.ReplaceAllString(line, "")
line = strings.TrimSpace(line)
text.WriteString(line + "\n")
}
audio = append(audio, findAudioFromText(text.String())...)
y := findYearFromText(text.String(), title)
if y != "" {
year = y
}
size = append(size, findSizesFromText(text.String())...)
})
// find any link from imdb
imdbLink := ""
article.Find("a").Each(func(i int, s *goquery.Selection) {
link, _ := s.Attr("href")
_imdbLink, err := getIMDBLink(link)
if err == nil {
imdbLink = _imdbLink
}
})
size = stableUniq(size)
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
// for each magnet link, create a new indexed torrent
for it, magnetLink := range magnetLinks {
it := it
go func(it int, magnetLink string) {
magnet, err := magnet.ParseMagnetUri(magnetLink)
if err != nil {
fmt.Println(err)
}
releaseTitle := magnet.DisplayName
infoHash := magnet.InfoHash.String()
trackers := magnet.Trackers
magnetAudio := getAudioFromTitle(releaseTitle, audio)
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
if err != nil {
fmt.Println(err)
}
title := processTitle(title, magnetAudio)
// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
var mySize string
if len(size) == len(magnetLinks) {
mySize = size[it]
}
ixt := schema.IndexedTorrent{
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
OriginalTitle: title,
Details: link,
Year: year,
IMDB: imdbLink,
Audio: magnetAudio,
MagnetLink: magnetLink,
Date: date,
InfoHash: infoHash,
Trackers: trackers,
LeechCount: peer,
SeedCount: seed,
Size: mySize,
}
chanIndexedTorrent <- ixt
}(it, magnetLink)
}
for i := 0; i < len(magnetLinks); i++ {
it := <-chanIndexedTorrent
indexedTorrents = append(indexedTorrents, it)
}
return indexedTorrents, nil
}

View File

@@ -4,6 +4,7 @@ import (
"encoding/json" "encoding/json"
"net/http" "net/http"
"strconv" "strconv"
"time"
meilisearch "github.com/felipemarinho97/torrent-indexer/search" meilisearch "github.com/felipemarinho97/torrent-indexer/search"
) )
@@ -13,6 +14,23 @@ type MeilisearchHandler struct {
Module *meilisearch.SearchIndexer Module *meilisearch.SearchIndexer
} }
// HealthResponse represents the health check response
type HealthResponse struct {
Status string `json:"status"`
Service string `json:"service"`
Details map[string]interface{} `json:"details,omitempty"`
Timestamp string `json:"timestamp"`
}
// StatsResponse represents the stats endpoint response
type StatsResponse struct {
Status string `json:"status"`
NumberOfDocuments int64 `json:"numberOfDocuments"`
IsIndexing bool `json:"isIndexing"`
FieldDistribution map[string]int64 `json:"fieldDistribution"`
Service string `json:"service"`
}
// NewMeilisearchHandler creates a new instance of MeilisearchHandler. // NewMeilisearchHandler creates a new instance of MeilisearchHandler.
func NewMeilisearchHandler(module *meilisearch.SearchIndexer) *MeilisearchHandler { func NewMeilisearchHandler(module *meilisearch.SearchIndexer) *MeilisearchHandler {
return &MeilisearchHandler{Module: module} return &MeilisearchHandler{Module: module}
@@ -53,3 +71,88 @@ func (h *MeilisearchHandler) SearchTorrentHandler(w http.ResponseWriter, r *http
http.Error(w, "Failed to encode response", http.StatusInternalServerError) http.Error(w, "Failed to encode response", http.StatusInternalServerError)
} }
} }
// HealthHandler provides a health check endpoint for Meilisearch.
func (h *MeilisearchHandler) HealthHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
w.Header().Set("Content-Type", "application/json")
// Check if Meilisearch is healthy
isHealthy := h.Module.IsHealthy()
response := HealthResponse{
Service: "meilisearch",
Timestamp: getCurrentTimestamp(),
}
if isHealthy {
// Try to get additional stats for more detailed health info
stats, err := h.Module.GetStats()
if err == nil {
response.Status = "healthy"
response.Details = map[string]interface{}{
"documents": stats.NumberOfDocuments,
"indexing": stats.IsIndexing,
}
w.WriteHeader(http.StatusOK)
} else {
// Service is up but can't get stats
response.Status = "degraded"
response.Details = map[string]interface{}{
"error": "Could not retrieve stats",
}
w.WriteHeader(http.StatusOK)
}
} else {
// Service is down
response.Status = "unhealthy"
w.WriteHeader(http.StatusServiceUnavailable)
}
if err := json.NewEncoder(w).Encode(response); err != nil {
http.Error(w, "Failed to encode response", http.StatusInternalServerError)
}
}
// StatsHandler provides detailed statistics about the Meilisearch index.
func (h *MeilisearchHandler) StatsHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
w.Header().Set("Content-Type", "application/json")
// Get detailed stats from Meilisearch
stats, err := h.Module.GetStats()
if err != nil {
// Check if it's a connectivity issue
if !h.Module.IsHealthy() {
http.Error(w, "Meilisearch service is unavailable", http.StatusServiceUnavailable)
return
}
http.Error(w, "Failed to retrieve statistics", http.StatusInternalServerError)
return
}
response := StatsResponse{
Status: "healthy",
Service: "meilisearch",
NumberOfDocuments: stats.NumberOfDocuments,
IsIndexing: stats.IsIndexing,
FieldDistribution: stats.FieldDistribution,
}
if err := json.NewEncoder(w).Encode(response); err != nil {
http.Error(w, "Failed to encode response", http.StatusInternalServerError)
}
}
// getCurrentTimestamp returns the current timestamp in RFC3339 format
func getCurrentTimestamp() string {
return time.Now().Format(time.RFC3339)
}

View File

@@ -140,6 +140,8 @@ func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]sch
return nil, err return nil, err
} }
date := getPublishedDateFromRawString(link)
post := doc.Find(".post") post := doc.Find(".post")
capa := post.Find(".capa") capa := post.Find(".capa")
title := capa.Find(".post-description > h2").Text() title := capa.Find(".post-description > h2").Text()
@@ -171,7 +173,6 @@ func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]sch
text.WriteString(span.Text()) text.WriteString(span.Text())
text.WriteString(" ") text.WriteString(" ")
}) })
fmt.Println(text.String())
audio = append(audio, findAudioFromText(text.String())...) audio = append(audio, findAudioFromText(text.String())...)
y := findYearFromText(text.String(), title) y := findYearFromText(text.String(), title)
if y != "" { if y != "" {
@@ -195,22 +196,23 @@ func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]sch
if err != nil { if err != nil {
fmt.Println(err) fmt.Println(err)
} }
releaseTitle := magnet.DisplayName releaseTitle := strings.TrimSpace(magnet.DisplayName)
// url decode the title
releaseTitle, err = url.QueryUnescape(releaseTitle)
if err != nil {
fmt.Println(err)
releaseTitle = strings.TrimSpace(magnet.DisplayName)
}
infoHash := magnet.InfoHash.String() infoHash := magnet.InfoHash.String()
trackers := magnet.Trackers trackers := magnet.Trackers
magnetAudio := []schema.Audio{} for i, tracker := range trackers {
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") { unescapedTracker, err := url.QueryUnescape(tracker)
magnetAudio = append(magnetAudio, audio...) if err != nil {
} else if len(audio) > 1 { fmt.Println(err)
// remove portuguese audio, and append to magnetAudio
for _, a := range audio {
if a != schema.AudioPortuguese {
magnetAudio = append(magnetAudio, a)
} }
trackers[i] = strings.TrimSpace(unescapedTracker)
} }
} else { magnetAudio := getAudioFromTitle(releaseTitle, audio)
magnetAudio = append(magnetAudio, audio...)
}
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers) peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
if err != nil { if err != nil {
@@ -233,7 +235,7 @@ func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]sch
IMDB: imdbLink, IMDB: imdbLink,
Audio: magnetAudio, Audio: magnetAudio,
MagnetLink: magnetLink, MagnetLink: magnetLink,
Date: time.Now(), Date: date,
InfoHash: infoHash, InfoHash: infoHash,
Trackers: trackers, Trackers: trackers,
LeechCount: peer, LeechCount: peer,

View File

@@ -143,7 +143,7 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
article := doc.Find("article") article := doc.Find("article")
title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1) title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1)
textContent := article.Find("div.content") textContent := article.Find("div.content")
date := getPublishedDateTDF(doc) date := getPublishedDateFromMeta(doc)
magnets := textContent.Find("a[href^=\"magnet\"]") magnets := textContent.Find("a[href^=\"magnet\"]")
var magnetLinks []string var magnetLinks []string
magnets.Each(func(i int, s *goquery.Selection) { magnets.Each(func(i int, s *goquery.Selection) {
@@ -206,19 +206,7 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
releaseTitle := magnet.DisplayName releaseTitle := magnet.DisplayName
infoHash := magnet.InfoHash.String() infoHash := magnet.InfoHash.String()
trackers := magnet.Trackers trackers := magnet.Trackers
magnetAudio := []schema.Audio{} magnetAudio := getAudioFromTitle(releaseTitle, audio)
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
magnetAudio = append(magnetAudio, audio...)
} else if len(audio) > 1 {
// remove portuguese audio, and append to magnetAudio
for _, a := range audio {
if a != schema.AudioPortuguese {
magnetAudio = append(magnetAudio, a)
}
}
} else {
magnetAudio = append(magnetAudio, audio...)
}
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers) peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
if err != nil { if err != nil {
@@ -259,15 +247,3 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
return indexedTorrents, nil return indexedTorrents, nil
} }
func getPublishedDateTDF(document *goquery.Document) time.Time {
var date time.Time
//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">
datePublished := strings.TrimSpace(document.Find("meta[property=\"article:published_time\"]").AttrOr("content", ""))
if datePublished != "" {
date, _ = time.Parse(time.RFC3339, datePublished)
}
return date
}

View File

@@ -46,13 +46,16 @@ func main() {
metricsMux := http.NewServeMux() metricsMux := http.NewServeMux()
indexerMux.HandleFunc("/", handler.HandlerIndex) indexerMux.HandleFunc("/", handler.HandlerIndex)
indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
indexerMux.HandleFunc("/indexers/torrent-dos-filmes", indexers.HandlerTorrentDosFilmesIndexer)
indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer) indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
indexerMux.HandleFunc("/indexers/comandohds", indexers.HandlerComandoHDsIndexer) indexerMux.HandleFunc("/indexers/comandohds", indexers.HandlerComandoHDsIndexer)
indexerMux.HandleFunc("/indexers/rede_torrent", indexers.HandlerRedeTorrentIndexer)
indexerMux.HandleFunc("/indexers/starck-filmes", indexers.HandlerStarckFilmesIndexer) indexerMux.HandleFunc("/indexers/starck-filmes", indexers.HandlerStarckFilmesIndexer)
indexerMux.HandleFunc("/indexers/torrent-dos-filmes", indexers.HandlerTorrentDosFilmesIndexer)
indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer) indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer)
indexerMux.HandleFunc("/search", search.SearchTorrentHandler) indexerMux.HandleFunc("/search", search.SearchTorrentHandler)
indexerMux.HandleFunc("/search/health", search.HealthHandler)
indexerMux.HandleFunc("/search/stats", search.StatsHandler)
indexerMux.Handle("/ui/", http.StripPrefix("/ui/", http.FileServer(http.FS(public.UIFiles)))) indexerMux.Handle("/ui/", http.StripPrefix("/ui/", http.FileServer(http.FS(public.UIFiles))))
metricsMux.Handle("/metrics", promhttp.Handler()) metricsMux.Handle("/metrics", promhttp.Handler())
@@ -66,7 +69,7 @@ func main() {
port := os.Getenv("PORT") port := os.Getenv("PORT")
if port == "" { if port == "" {
port = "7006" port = "7007"
} }
fmt.Printf("Server listening on :%s\n", port) fmt.Printf("Server listening on :%s\n", port)

View File

@@ -9,8 +9,8 @@
<script src="https://cdn.jsdelivr.net/npm/@heroicons/react/solid@2.0.0/dist/index.umd.js"></script> <script src="https://cdn.jsdelivr.net/npm/@heroicons/react/solid@2.0.0/dist/index.umd.js"></script>
</head> </head>
<body class="bg-gray-900 text-white font-sans"> <body class="bg-gray-900 text-white font-sans min-h-screen flex flex-col">
<div class="container mx-auto p-6"> <div class="container mx-auto p-6 flex-grow">
<!-- Header --> <!-- Header -->
<header class="text-center mb-10"> <header class="text-center mb-10">
<h1 class="text-4xl font-bold text-blue-400">Torrent Indexer 🇧🇷</h1> <h1 class="text-4xl font-bold text-blue-400">Torrent Indexer 🇧🇷</h1>
@@ -26,12 +26,93 @@
</div> </div>
<!-- Results Section --> <!-- Results Section -->
<div id="results" class="space-y-6"> <div id="results" class="space-y-6 mb-10">
<!-- Dynamic content will be injected here --> <!-- Dynamic content will be injected here -->
</div> </div>
<!-- Health Warning -->
<div id="health-warning" class="hidden mb-6 p-4 bg-yellow-800 border border-yellow-600 rounded-lg">
<div class="flex items-center gap-2">
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5 text-yellow-400" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 9v2m0 4h.01m-9 4h18a2 2 0 002-2V7a2 2 0 00-2-2H3a2 2 0 00-2 2v10a2 2 0 002 2z" />
</svg>
<span class="font-bold text-yellow-400">Service Warning</span>
</div>
<p class="text-yellow-200 mt-2">Search functionality may be disabled or experiencing issues. Please try again later.</p>
</div>
</div>
<!-- Stats Section -->
<div id="database-statistics" class="stats-info mt-auto mb-1 p-3 rounded text-center">
<span id="torrentStats" class="text-gray-400">Loading stats...</span>
</div> </div>
<script> <script>
// Global variables
let serviceHealthy = true;
// Function to check service health
async function checkHealth() {
try {
const response = await fetch('/search/health');
const health = await response.json();
if (response.status === 503 || health.status === 'unhealthy') {
serviceHealthy = false;
showHealthWarning();
hideDatabaseStatistics();
} else if (health.status === 'degraded') {
serviceHealthy = true; // Still operational
showHealthWarning(); // But show warning
} else {
serviceHealthy = true;
hideHealthWarning();
}
} catch (error) {
serviceHealthy = false;
showHealthWarning();
console.error('Health check failed:', error);
}
}
// Function to show health warning
function showHealthWarning() {
document.getElementById('health-warning').classList.remove('hidden');
}
// Function to hide health warning
function hideHealthWarning() {
document.getElementById('health-warning').classList.add('hidden');
}
function hideDatabaseStatistics() {
document.getElementById('database-statistics').classList.add('hidden');
}
// Function to load stats
async function loadStats() {
try {
const response = await fetch('/search/stats');
if (response.ok) {
const stats = await response.json();
const statsElement = document.getElementById('torrentStats');
const formattedStats = `
<span class="text-sm text-gray-500">
<span class="text-green-400 font-medium">${stats.numberOfDocuments?.toLocaleString()+'+' || 'N/A'}</span> indexed torrents!
</span>
`;
statsElement.innerHTML = formattedStats;
} else {
throw new Error('Failed to load stats');
}
} catch (error) {
hideDatabaseStatistics();
console.error('Stats loading failed:', error);
}
}
// Function to render a single torrent result // Function to render a single torrent result
function renderTorrent(torrent) { function renderTorrent(torrent) {
return ` return `
@@ -100,27 +181,45 @@
const results = await response.json(); const results = await response.json();
const resultsContainer = document.getElementById('results'); const resultsContainer = document.getElementById('results');
if (results.length === 0) {
resultsContainer.innerHTML = `
<div class="p-6 bg-gray-800 rounded-lg shadow-md text-center">
<p class="text-xl font-bold text-gray-400">No results found</p>
<p class="text-gray-500 mt-2">Try different search terms or check spelling</p>
</div>
`;
} else {
resultsContainer.innerHTML = results.map(renderTorrent).join(''); resultsContainer.innerHTML = results.map(renderTorrent).join('');
}
} catch (error) { } catch (error) {
// add error element
document.getElementById('results').innerHTML = ` document.getElementById('results').innerHTML = `
<div class="p-6 bg-red-800 rounded-lg shadow-md text-center"> <div class="p-6 bg-red-800 rounded-lg shadow-md text-center">
<p class="text-xl font-bold text-red-400">Error fetching search results</p> <p class="text-xl font-bold text-red-400">Error fetching search results</p>
<p class="text-gray-400 mt-2">Please try again later.</p> <p class="text-gray-400 mt-2">Please try again later.</p>
</div> </div>
`; `;
//alert('Error fetching search results. Please try again.');
console.error(error); console.error(error);
} }
} }
// Event listeners
document.getElementById('search-btn').addEventListener('click', onSearch); document.getElementById('search-btn').addEventListener('click', onSearch);
// on enter press
document.getElementById('search-query').addEventListener('keypress', (e) => { document.getElementById('search-query').addEventListener('keypress', (e) => {
if (e.key === 'Enter') { if (e.key === 'Enter') {
onSearch(); onSearch();
} }
}); });
// Initialize page
document.addEventListener('DOMContentLoaded', async () => {
await checkHealth();
await loadStats();
// Refresh health and stats periodically
setInterval(checkHealth, 30000); // Check health every 30 seconds
setInterval(loadStats, 60000); // Update stats every minute
});
</script> </script>
</body> </body>

View File

@@ -19,6 +19,18 @@ type SearchIndexer struct {
IndexName string IndexName string
} }
// IndexStats represents statistics about the Meilisearch index
type IndexStats struct {
NumberOfDocuments int64 `json:"numberOfDocuments"`
IsIndexing bool `json:"isIndexing"`
FieldDistribution map[string]int64 `json:"fieldDistribution"`
}
// HealthStatus represents the health status of Meilisearch
type HealthStatus struct {
Status string `json:"status"`
}
// NewSearchIndexer creates a new instance of SearchIndexer. // NewSearchIndexer creates a new instance of SearchIndexer.
func NewSearchIndexer(baseURL, apiKey, indexName string) *SearchIndexer { func NewSearchIndexer(baseURL, apiKey, indexName string) *SearchIndexer {
return &SearchIndexer{ return &SearchIndexer{
@@ -145,3 +157,67 @@ func (t *SearchIndexer) SearchTorrent(query string, limit int) ([]schema.Indexed
return result.Hits, nil return result.Hits, nil
} }
// GetStats retrieves statistics about the Meilisearch index including document count.
// This method can be used for health checks and monitoring.
func (t *SearchIndexer) GetStats() (*IndexStats, error) {
url := fmt.Sprintf("%s/indexes/%s/stats", t.BaseURL, t.IndexName)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
if t.APIKey != "" {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
}
resp, err := t.Client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to execute request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("failed to get stats: status %d, body: %s", resp.StatusCode, body)
}
var stats IndexStats
if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
return nil, fmt.Errorf("failed to parse stats response: %w", err)
}
return &stats, nil
}
// IsHealthy checks if Meilisearch is available and responsive.
// Returns true if the service is healthy, false otherwise.
func (t *SearchIndexer) IsHealthy() bool {
url := fmt.Sprintf("%s/health", t.BaseURL)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return false
}
// Use a shorter timeout for health checks
client := &http.Client{Timeout: 5 * time.Second}
resp, err := client.Do(req)
if err != nil {
return false
}
defer resp.Body.Close()
return resp.StatusCode == http.StatusOK
}
// GetDocumentCount returns the number of indexed documents.
// This is a convenience method that extracts just the document count from stats.
func (t *SearchIndexer) GetDocumentCount() (int64, error) {
stats, err := t.GetStats()
if err != nil {
return 0, err
}
return stats.NumberOfDocuments, nil
}