darklyn/torrent-indexer
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Feat/Flaresolverr support (#12)

Browse Source 
* new: feat: add flaresolverr support

* chg: feat: add session pool

* chg: fix: deadlock error

* chg: fix: make it work without flaresolverr
This commit is contained in:
Felipe Marinho
2024-09-24 18:31:58 -03:00
committed by GitHub
parent ced533cd40
commit a6a848b284
9 changed files with 357 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
api/bludv.go
View File

@@ -44,7 +44,7 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
}
fmt.Println("URL:>", url)
resp, err := http.Get(url)
resp, err := i.requester.GetDocument(ctx, url)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -54,9 +54,9 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
i.metrics.IndexerErrors.WithLabelValues("bludv").Inc()
return
}
defer resp.Body.Close()
defer resp.Close()
doc, err := goquery.NewDocumentFromReader(resp.Body)
doc, err := goquery.NewDocumentFromReader(resp)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})

13
api/comando_torrents.go
View File

@@ -60,7 +60,7 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
}
fmt.Println("URL:>", url)
resp, err := http.Get(url)
resp, err := i.requester.GetDocument(ctx, url)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -70,9 +70,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
i.metrics.IndexerErrors.WithLabelValues("comando").Inc()
return
}
defer resp.Body.Close()
defer resp.Close()
doc, err := goquery.NewDocumentFromReader(resp.Body)
doc, err := goquery.NewDocumentFromReader(resp)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -405,17 +405,18 @@ func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Documen
docCache, err := i.redis.Get(ctx, link)
if err == nil {
i.metrics.CacheHits.WithLabelValues("document_body").Inc()
fmt.Printf("returning from long-lived cache: %s\n", link)
return goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(docCache)))
}
defer i.metrics.CacheMisses.WithLabelValues("document_body").Inc()
resp, err := http.Get(link)
resp, err := i.requester.GetDocument(ctx, link)
if err != nil {
return nil, err
}
defer resp.Body.Close()
defer resp.Close()
body, err := io.ReadAll(resp.Body)
body, err := io.ReadAll(resp)
if err != nil {
return nil, err
}

13
api/index.go
View File

@@ -7,12 +7,14 @@ import (
"github.com/felipemarinho97/torrent-indexer/cache"
"github.com/felipemarinho97/torrent-indexer/monitoring"
"github.com/felipemarinho97/torrent-indexer/requester"
"github.com/felipemarinho97/torrent-indexer/schema"
)
type Indexer struct {
redis *cache.Redis
metrics *monitoring.Metrics
redis *cache.Redis
metrics *monitoring.Metrics
requester *requester.Requster
}
type IndexerMeta struct {
@@ -42,10 +44,11 @@ type IndexedTorrent struct {
Similarity float32 `json:"similarity"`
}
func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics) *Indexer {
func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster) *Indexer {
return &Indexer{
redis: redis,
metrics: metrics,
redis: redis,
metrics: metrics,
requester: req,
}
}