darklyn/torrent-indexer
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

chg: feat: rewrite to be a torrent indexer

Browse Source
This commit is contained in:
Marinho
2023-09-23 17:02:55 +00:00
parent c68df1ab37
commit faa721dc19
15 changed files with 711 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
api/b3/prices/[ticker].go
View File

@@ -1,143 +0,0 @@
package quotation
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
log "github.com/sirupsen/logrus"
)
type RawResponse struct {
BizSts struct {
Cd string `json:"cd"`
} `json:"BizSts"`
Msg struct {
DtTm string `json:"dtTm"`
} `json:"Msg"`
Trad []struct {
Scty struct {
SctyQtn struct {
OpngPric float64 `json:"opngPric"`
MinPric float64 `json:"minPric"`
MaxPric float64 `json:"maxPric"`
AvrgPric float64 `json:"avrgPric"`
CurPrc float64 `json:"curPrc"`
PrcFlcn float64 `json:"prcFlcn"`
} `json:"SctyQtn"`
Mkt struct {
Nm string `json:"nm"`
} `json:"mkt"`
Symb string `json:"symb"`
Desc string `json:"desc"`
IndxCmpnInd bool `json:"indxCmpnInd"`
} `json:"scty"`
TTLQty int `json:"ttlQty"`
} `json:"Trad"`
}
type Response struct {
Symbol string `json:"symbol"`
Name string `json:"name"`
Market string `json:"market"`
OpeningPrice float64 `json:"openingPrice"`
MinPrice float64 `json:"minPrice"`
MaxPrice float64 `json:"maxPrice"`
AveragePrice float64 `json:"averagePrice"`
CurrentPrice float64 `json:"currentPrice"`
PriceVariation float64 `json:"priceVariation"`
IndexComponentIndicator bool `json:"indexComponentIndicator"`
}
type RawErrorResponse struct {
BizSts struct {
Cd string `json:"cd"`
Desc string `json:"desc"`
} `json:"BizSts"`
Msg struct {
DtTm string `json:"dtTm"`
} `json:"Msg"`
}
type Error struct {
Message string `json:"message"`
}
func HandlerListCompanies(w http.ResponseWriter, r *http.Request) {
ticker := strings.Split(r.URL.Path, "/")[4]
log.Info("Getting quotation info for ticker: " + ticker)
if ticker == "" {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Ticker is required"))
return
}
client := http.Client{}
res, err := client.Get(fmt.Sprintf("https://cotacao.b3.com.br/mds/api/v1/instrumentQuotation/%s", ticker))
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
defer res.Body.Close()
w.Header().Set("Content-Type", "application/json")
// add 1min cache header
w.Header().Set("Cache-Control", "max-age=60, public")
out, err := ioutil.ReadAll(res.Body)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
var raw RawResponse
err = json.Unmarshal(out, &raw)
if err != nil || raw.BizSts.Cd != "OK" {
var errorResponse RawErrorResponse
err = json.Unmarshal(out, &errorResponse)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
// 404
w.WriteHeader(http.StatusNotFound)
formatedError := Error{Message: errorResponse.BizSts.Desc}
err := json.NewEncoder(w).Encode(formatedError)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
return
}
var response []Response
for _, trad := range raw.Trad {
response = append(response, Response{
Symbol: trad.Scty.Symb,
Name: trad.Scty.Desc,
Market: trad.Scty.Mkt.Nm,
OpeningPrice: trad.Scty.SctyQtn.OpngPric,
MinPrice: trad.Scty.SctyQtn.MinPric,
MaxPrice: trad.Scty.SctyQtn.MaxPric,
AveragePrice: trad.Scty.SctyQtn.AvrgPric,
CurrentPrice: trad.Scty.SctyQtn.CurPrc,
PriceVariation: trad.Scty.SctyQtn.PrcFlcn,
IndexComponentIndicator: trad.Scty.IndxCmpnInd,
})
}
err = json.NewEncoder(w).Encode(response[0])
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
return
}

187
api/indexers/comando_torrents.go → api/comando_torrents.go
View File

@@ -1,8 +1,11 @@
package indexers
package handler
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"regexp"
@@ -10,6 +13,8 @@ import (
"time"
"github.com/PuerkitoBio/goquery"
"github.com/felipemarinho97/torrent-indexer/schema"
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
)
const (
@@ -17,14 +22,6 @@ const (
queryFilter = "?s="
)
type Audio string
const (
AudioPortuguese = "Português"
AudioEnglish = "Inglês"
AudioSpanish = "Espanhol"
)
var replacer = strings.NewReplacer(
"janeiro", "01",
"fevereiro", "02",
@@ -41,28 +38,30 @@ var replacer = strings.NewReplacer(
)
type IndexedTorrent struct {
Title string `json:"title"`
OriginalTitle string `json:"original_title"`
Details string `json:"details"`
Year string `json:"year"`
Audio []Audio `json:"audio"`
MagnetLink string `json:"magnet_link"`
Date time.Time `json:"date"`
InfoHash string `json:"info_hash"`
Title string `json:"title"`
OriginalTitle string `json:"original_title"`
Details string `json:"details"`
Year string `json:"year"`
Audio []schema.Audio `json:"audio"`
MagnetLink string `json:"magnet_link"`
Date time.Time `json:"date"`
InfoHash string `json:"info_hash"`
Trackers []string `json:"trackers"`
LeechCount int `json:"leech_count"`
SeedCount int `json:"seed_count"`
}
func HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) {
func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
// supported query params: q, season, episode
q := r.URL.Query().Get("q")
if q == "" {
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(map[string]string{"error": "param q is required"})
return
}
// URL encode query param
q = url.QueryEscape(q)
url := URL + queryFilter + q
url := URL
if q != "" {
url = fmt.Sprintf("%s%s%s", URL, queryFilter, q)
}
fmt.Println("URL:>", url)
resp, err := http.Get(url)
@@ -92,7 +91,7 @@ func HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) {
var indexedTorrents []IndexedTorrent
for _, link := range links {
go func(link string) {
torrents, err := getTorrents(link)
torrents, err := getTorrents(ctx, i, link)
if err != nil {
fmt.Println(err)
errChan <- err
@@ -114,15 +113,9 @@ func HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(indexedTorrents)
}
func getTorrents(link string) ([]IndexedTorrent, error) {
func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
var indexedTorrents []IndexedTorrent
resp, err := http.Get(link)
if err != nil {
return nil, err
}
defer resp.Body.Close()
doc, err := goquery.NewDocumentFromReader(resp.Body)
doc, err := getDocument(ctx, i, link)
if err != nil {
return nil, err
}
@@ -153,7 +146,7 @@ func getTorrents(link string) ([]IndexedTorrent, error) {
magnetLinks = append(magnetLinks, magnetLink)
})
var audio []Audio
var audio []schema.Audio
var year string
article.Find("div.entry-content > p").Each(func(i int, s *goquery.Selection) {
// pattern:
@@ -165,6 +158,7 @@ func getTorrents(link string) ([]IndexedTorrent, error) {
// Formato: MKV
// Qualidade: WEB-DL
// Áudio: Português | Inglês
// Idioma: Português | Inglês
// Legenda: Português
// Tamanho: –
// Qualidade de Áudio: 10
@@ -173,18 +167,19 @@ func getTorrents(link string) ([]IndexedTorrent, error) {
// Servidor: Torrent
text := s.Text()
re := regexp.MustCompile(`Áudio: (.*)`)
//re := regexp.MustCompile(`Áudio: (.*)`)
re := regexp.MustCompile(`(Áudio|Idioma): (.*)`)
audioMatch := re.FindStringSubmatch(text)
if len(audioMatch) > 0 {
langs_raw := strings.Split(audioMatch[1], "|")
sep := getSeparator(audioMatch[2])
langs_raw := strings.Split(audioMatch[2], sep)
for _, lang := range langs_raw {
lang = strings.TrimSpace(lang)
if lang == "Português" {
audio = append(audio, AudioPortuguese)
} else if lang == "Inglês" {
audio = append(audio, AudioEnglish)
} else if lang == "Espanhol" {
audio = append(audio, AudioSpanish)
a := schema.GetAudioFromString(lang)
if a != nil {
audio = append(audio, *a)
} else {
fmt.Println("unknown language:", lang)
}
}
}
@@ -208,25 +203,30 @@ func getTorrents(link string) ([]IndexedTorrent, error) {
// for each magnet link, create a new indexed torrent
for _, magnetLink := range magnetLinks {
releaseTitle := extractReleaseName(magnetLink)
magnetAudio := []Audio{}
magnetAudio := []schema.Audio{}
if strings.Contains(strings.ToLower(releaseTitle), "dual") {
magnetAudio = append(magnetAudio, AudioPortuguese)
magnetAudio = append(magnetAudio, audio...)
} else {
// filter portuguese audio from list
for _, lang := range audio {
if lang != AudioPortuguese {
magnetAudio = append(magnetAudio, lang)
} else if len(audio) > 1 {
// remove portuguese audio, and append to magnetAudio
for _, a := range audio {
if a != schema.AudioPortuguese {
magnetAudio = append(magnetAudio, a)
}
}
} else {
magnetAudio = append(magnetAudio, audio...)
}
// remove duplicates
magnetAudio = removeDuplicates(magnetAudio)
// decode url encoded title
releaseTitle, _ = url.QueryUnescape(releaseTitle)
infoHash := extractInfoHash(magnetLink)
trackers := extractTrackers(magnetLink)
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, infoHash, trackers)
if err != nil {
fmt.Println(err)
}
title := processTitle(title, magnetAudio)
indexedTorrents = append(indexedTorrents, IndexedTorrent{
Title: releaseTitle,
@@ -237,12 +237,75 @@ func getTorrents(link string) ([]IndexedTorrent, error) {
MagnetLink: magnetLink,
Date: date,
InfoHash: infoHash,
Trackers: trackers,
LeechCount: peer,
SeedCount: seed,
})
}
return indexedTorrents, nil
}
func processTitle(title string, a []schema.Audio) string {
// remove ' - Donwload' from title
title = strings.Replace(title, " - Download", "", -1)
// remove 'comando.la' from title
title = strings.Replace(title, "comando.la", "", -1)
// add audio ISO 639-2 code to title between ()
if len(a) > 0 {
audio := []string{}
for _, lang := range a {
audio = append(audio, lang.String())
}
title = fmt.Sprintf("%s (%s)", title, strings.Join(audio, ", "))
}
return title
}
func getSeparator(s string) string {
if strings.Contains(s, "|") {
return "|"
} else if strings.Contains(s, ",") {
return ","
}
return " "
}
func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Document, error) {
// try to get from redis first
docCache, err := i.redis.Get(ctx, link)
if err == nil {
return goquery.NewDocumentFromReader(ioutil.NopCloser(bytes.NewReader(docCache)))
}
resp, err := http.Get(link)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
// set cache
err = i.redis.Set(ctx, link, body)
if err != nil {
fmt.Println(err)
}
doc, err := goquery.NewDocumentFromReader(ioutil.NopCloser(bytes.NewReader(body)))
if err != nil {
return nil, err
}
return doc, nil
}
func extractReleaseName(magnetLink string) string {
re := regexp.MustCompile(`dn=(.*?)&`)
matches := re.FindStringSubmatch(magnetLink)
@@ -261,16 +324,14 @@ func extractInfoHash(magnetLink string) string {
return ""
}
func removeDuplicates(elements []Audio) []Audio {
encountered := map[Audio]bool{}
result := []Audio{}
for _, element := range elements {
if !encountered[element] {
encountered[element] = true
result = append(result, element)
}
func extractTrackers(magnetLink string) []string {
re := regexp.MustCompile(`tr=(.*?)&`)
matches := re.FindAllStringSubmatch(magnetLink, -1)
var trackers []string
for _, match := range matches {
// url decode
tracker, _ := url.QueryUnescape(match[1])
trackers = append(trackers, tracker)
}
return result
return trackers
}

33
api/index.go
View File

@@ -1,16 +1,37 @@
package handler
import (
"fmt"
"encoding/json"
"net/http"
"time"
"github.com/felipemarinho97/torrent-indexer/cache"
)
type Indexer struct {
redis *cache.Redis
}
func NewIndexers(redis *cache.Redis) *Indexer {
return &Indexer{
redis: redis,
}
}
func HandlerIndex(w http.ResponseWriter, r *http.Request) {
currentTime := time.Now().Format(time.RFC850)
fmt.Fprintf(w, currentTime)
w.Header().Set("Content-Type", "text/html")
fmt.Fprintf(w, `
Github OAuth2 => <a href="https://github.com/xjh22222228/github-oauth2" target="_blank">https://github.com/xjh22222228/github-oauth2</a>
`)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]interface{}{
"time": currentTime,
"endpoints": map[string]interface{}{
"/indexers/comando_torrents": map[string]interface{}{
"method": "GET",
"description": "Indexer for comando torrents",
"query_params": map[string]string{
"q": "search query",
},
},
},
})
}

39
api/statusinvest/companies.go
View File

@@ -1,39 +0,0 @@
package statusinvest
import (
"io/ioutil"
"net/http"
)
func HandlerListCompanies(w http.ResponseWriter, r *http.Request) {
client := http.Client{}
req, err := http.NewRequest("GET", "https://statusinvest.com.br/acao/companiesnavigation?page=1&size=500", nil)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")
req.Header.Set("Accept", "application/json, text/plain, */*")
resp, err := client.Do(req)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
defer resp.Body.Close()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
w.Write(body)
}

85
api/statusinvest/indicators.go
View File

@@ -1,85 +0,0 @@
package statusinvest
import (
"encoding/json"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
type Response struct {
Success bool `json:"success"`
Data map[string][]map[string]interface{} `json:"data"`
}
type ParsedResponse map[string]interface{}
func HandlerIndicators(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
w.WriteHeader(http.StatusMethodNotAllowed)
w.Write([]byte("Method not allowed"))
return
}
time := r.URL.Query().Get("time")
if time == "" {
time = "7"
}
ticker := r.URL.Query().Get("ticker")
if ticker == "" {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Ticker is required"))
return
}
data := url.Values{
"codes[]": strings.Split(ticker, ","),
"time": {time},
"byQuarter": {"false"},
"futureData": {"false"},
}
resp, err := http.PostForm("https://statusinvest.com.br/acao/indicatorhistoricallist", data)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
defer resp.Body.Close()
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(resp.StatusCode)
out, err := ioutil.ReadAll(resp.Body)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
var indicators Response
err = json.Unmarshal(out, &indicators)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
parsedResponse := ParsedResponse{}
d := indicators.Data[ticker]
for _, v := range d {
v := v
parsedResponse[v["key"].(string)] = v
}
out, err = json.Marshal(parsedResponse)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
w.Write(out)
}