Feat/Flaresolverr support (#12)

* new: feat: add flaresolverr support

* chg: feat: add session pool

* chg: fix: deadlock error

* chg: fix: make it work without flaresolverr
This commit is contained in:
2024-09-24 18:31:58 -03:00
committed by GitHub
parent ced533cd40
commit a6a848b284
9 changed files with 357 additions and 21 deletions

View File

@@ -44,7 +44,7 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
}
fmt.Println("URL:>", url)
resp, err := http.Get(url)
resp, err := i.requester.GetDocument(ctx, url)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
@@ -54,9 +54,9 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
i.metrics.IndexerErrors.WithLabelValues("bludv").Inc()
return
}
defer resp.Body.Close()
defer resp.Close()
doc, err := goquery.NewDocumentFromReader(resp.Body)
doc, err := goquery.NewDocumentFromReader(resp)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})