Feat/Flaresolverr support (#12)
* new: feat: add flaresolverr support * chg: feat: add session pool * chg: fix: deadlock error * chg: fix: make it work without flaresolverr
This commit is contained in:
GitHub
230
requester/flaresolverr.go
Normal file
230
requester/flaresolverr.go
Normal file
@@ -0,0 +1,230 @@
|
||||
package requester
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type FlareSolverr struct {
|
||||
url string
|
||||
maxTimeout int
|
||||
httpClient *http.Client
|
||||
sessionPool chan string
|
||||
mu sync.Mutex
|
||||
initiated bool
|
||||
}
|
||||
|
||||
func NewFlareSolverr(url string, timeoutMilli int) *FlareSolverr {
|
||||
poolSize := 5
|
||||
httpClient := &http.Client{}
|
||||
sessionPool := make(chan string, poolSize) // Pool size of 5 sessions
|
||||
|
||||
f := &FlareSolverr{
|
||||
url: url,
|
||||
maxTimeout: timeoutMilli,
|
||||
httpClient: httpClient,
|
||||
sessionPool: sessionPool,
|
||||
}
|
||||
|
||||
err := f.FillSessionPool()
|
||||
if err == nil {
|
||||
f.initiated = true
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *FlareSolverr) FillSessionPool() error {
|
||||
// Check if the pool is already filled
|
||||
if len(f.sessionPool) == cap(f.sessionPool) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pre-initialize the pool with existing sessions
|
||||
sessions, err := f.ListSessions()
|
||||
if err != nil {
|
||||
fmt.Println("Failed to list existing FlareSolverr sessions:", err)
|
||||
return err
|
||||
} else {
|
||||
for _, session := range sessions {
|
||||
// Add available sessions to the pool
|
||||
if len(f.sessionPool) < cap(f.sessionPool) {
|
||||
f.sessionPool <- session
|
||||
}
|
||||
}
|
||||
if len(f.sessionPool) > 0 {
|
||||
fmt.Printf("Added %d FlareSolverr sessions to the pool\n", len(f.sessionPool))
|
||||
}
|
||||
}
|
||||
|
||||
// If fewer than poolSize sessions were found, create new ones to fill the pool
|
||||
for len(f.sessionPool) < cap(f.sessionPool) {
|
||||
f.CreateSession()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FlareSolverr) CreateSession() string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
body := map[string]string{"cmd": "sessions.create"}
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := f.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
var sessionResponse map[string]interface{}
|
||||
err = json.NewDecoder(resp.Body).Decode(&sessionResponse)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
session := sessionResponse["session"].(string)
|
||||
// Add session to the pool
|
||||
f.sessionPool <- session
|
||||
|
||||
fmt.Println("Created new FlareSolverr session:", session)
|
||||
return session
|
||||
}
|
||||
|
||||
func (f *FlareSolverr) ListSessions() ([]string, error) {
|
||||
body := map[string]string{"cmd": "sessions.list"}
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := f.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
var sessionsResponse map[string]interface{}
|
||||
err = json.NewDecoder(resp.Body).Decode(&sessionsResponse)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sessions := sessionsResponse["sessions"].([]interface{})
|
||||
var sessionIDs []string
|
||||
for _, session := range sessions {
|
||||
sessionIDs = append(sessionIDs, session.(string))
|
||||
}
|
||||
|
||||
return sessionIDs, nil
|
||||
}
|
||||
|
||||
func (f *FlareSolverr) RetrieveSession() string {
|
||||
// Blocking receive from the session pool.
|
||||
session := <-f.sessionPool
|
||||
return session
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
Solution struct {
|
||||
Url string `json:"url"`
|
||||
Status int `json:"status"`
|
||||
Cookies []struct {
|
||||
Domain string `json:"domain"`
|
||||
Expiry int `json:"expiry"`
|
||||
HttpOnly bool `json:"httpOnly"`
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
SameSite string `json:"sameSite"`
|
||||
Secure bool `json:"secure"`
|
||||
Value string `json:"value"`
|
||||
} `json:"cookies"`
|
||||
UserAgent string `json:"userAgent"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
Response string `json:"response"`
|
||||
} `json:"solution"`
|
||||
}
|
||||
|
||||
func (f *FlareSolverr) Get(url string) (io.ReadCloser, error) {
|
||||
// Check if the FlareSolverr instance was initiated
|
||||
if !f.initiated {
|
||||
return io.NopCloser(bytes.NewReader([]byte(""))), nil
|
||||
}
|
||||
|
||||
// Retrieve session from the pool (blocking if no sessions available)
|
||||
session := f.RetrieveSession()
|
||||
|
||||
// Ensure the session is returned to the pool after the request is done
|
||||
defer func() {
|
||||
f.sessionPool <- session
|
||||
}()
|
||||
|
||||
body := map[string]string{
|
||||
"cmd": "request.get",
|
||||
"url": url,
|
||||
"maxTimeout": fmt.Sprintf("%d", f.maxTimeout),
|
||||
"session": session,
|
||||
}
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := f.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Parse the response
|
||||
var response Response
|
||||
err = json.NewDecoder(resp.Body).Decode(&response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check if the response was successful
|
||||
if response.Status != "ok" {
|
||||
return nil, fmt.Errorf("failed to get response: %s", response.Message)
|
||||
}
|
||||
|
||||
// Check if "Under attack" is in the response
|
||||
if strings.Contains(response.Solution.Response, "Under attack") {
|
||||
return nil, fmt.Errorf("under attack")
|
||||
}
|
||||
|
||||
// Return the response body
|
||||
return io.NopCloser(bytes.NewReader([]byte(response.Solution.Response))), nil
|
||||
}
|
||||
93
requester/requester.go
Normal file
93
requester/requester.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package requester
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/cache"
|
||||
)
|
||||
|
||||
const (
|
||||
shortLivedCacheExpiration = 30 * time.Minute
|
||||
cacheKey = "shortLivedCache"
|
||||
)
|
||||
|
||||
var challangeRegex = regexp.MustCompile(`(?i)(just a moment|cf-chl-bypass|under attack)`)
|
||||
|
||||
type Requster struct {
|
||||
fs *FlareSolverr
|
||||
c *cache.Redis
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func NewRequester(fs *FlareSolverr, c *cache.Redis) *Requster {
|
||||
return &Requster{fs: fs, httpClient: &http.Client{}, c: c}
|
||||
}
|
||||
|
||||
func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser, error) {
|
||||
var body io.ReadCloser
|
||||
|
||||
// try request from short-lived cache
|
||||
key := fmt.Sprintf("%s:%s", cacheKey, url)
|
||||
bodyByte, err := i.c.Get(ctx, key)
|
||||
if err == nil {
|
||||
fmt.Printf("returning from short-lived cache: %s\n", url)
|
||||
body = io.NopCloser(bytes.NewReader(bodyByte))
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// try request with plain client
|
||||
resp, err := i.httpClient.Get(url)
|
||||
if err != nil {
|
||||
// try request with flare solverr
|
||||
body, err = i.fs.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
|
||||
}
|
||||
} else {
|
||||
defer resp.Body.Close()
|
||||
body = resp.Body
|
||||
}
|
||||
|
||||
bodyByte, err = io.ReadAll(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body: %w", err)
|
||||
}
|
||||
if hasChallange(bodyByte) {
|
||||
// try request with flare solverr
|
||||
body, err = i.fs.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
|
||||
}
|
||||
bodyByte, err = io.ReadAll(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body: %w", err)
|
||||
}
|
||||
fmt.Printf("request served from flaresolverr: %s\n", url)
|
||||
} else {
|
||||
fmt.Printf("request served from plain client: %s\n", url)
|
||||
}
|
||||
|
||||
// save response to cache if it's not a challange and body is not empty
|
||||
if !hasChallange(bodyByte) && len(bodyByte) > 0 {
|
||||
err = i.c.SetWithExpiration(ctx, key, bodyByte, shortLivedCacheExpiration)
|
||||
if err != nil {
|
||||
fmt.Printf("failed to save response to cache: %v\n", err)
|
||||
}
|
||||
fmt.Printf("saved to cache: %s\n", url)
|
||||
} else {
|
||||
return nil, fmt.Errorf("response is a challange")
|
||||
}
|
||||
|
||||
return io.NopCloser(bytes.NewReader(bodyByte)), nil
|
||||
}
|
||||
|
||||
// hasChallange checks if the body contains a challange by regex matching
|
||||
func hasChallange(body []byte) bool {
|
||||
return challangeRegex.Match(body)
|
||||
}
|
||||
Reference in New Issue
Block a user