From a6c3d3eed1155bbc5869b23b89fcd6b613e992a7 Mon Sep 17 00:00:00 2001 From: Felipe Marinho Date: Wed, 11 Dec 2024 16:01:09 -0300 Subject: [PATCH] Feat/ttl config (#23) * new: feat: add option to customize ttl * chg: docs: add docs form ENVs --- README.md | 13 +++++++++++++ cache/redis.go | 12 +++++++++--- go.mod | 1 + go.sum | 2 ++ main.go | 27 +++++++++++++++++++++++++-- requester/requester.go | 18 +++++++++++------- 6 files changed, 61 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8437b4a..d2318d2 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,19 @@ docker-compose up -d The server will be available at [http://localhost:8080/](http://localhost:8080/). +## Configuration + +You can configure the server using the following environment variables: + +- `PORT`: (optional) The port that the server will listen to. Default: `7006` +- `FLARESOLVERR_ADDRESS`: (optional) The address of the FlareSolverr instance. Default: `N/A` +- `REDIS_HOST`: (optional) The address of the Redis instance. Default: `localhost` +- `SHORT_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the short-lived cache in duration format. Default: `30m` + - This cache is used to cache homepage or search results. + - Example: `30m`, `1h`, `1h30m`, `1h30m30s` +- `LONG_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the long-lived cache in duration format. Default: `7d` + - This cache is used to store the torrent webpages (posts). You can set it to a higher value because the torrent pages are not updated frequently. + ## Integrating with Jackett You can integrate this indexer with Jackett by adding a new Torznab custom indexer. Here is an example of how to do it for the `bludv` indexer: diff --git a/cache/redis.go b/cache/redis.go index 85736e9..62e82dc 100644 --- a/cache/redis.go +++ b/cache/redis.go @@ -9,13 +9,14 @@ import ( "github.com/redis/go-redis/v9" ) -var ( +const ( DefaultExpiration = 24 * time.Hour * 7 // 7 days IndexerComandoTorrents = "indexer:comando_torrents" ) type Redis struct { - client *redis.Client + client *redis.Client + defaultExpiration time.Duration } func NewRedis() *Redis { @@ -28,15 +29,20 @@ func NewRedis() *Redis { Addr: fmt.Sprintf("%s:6379", redisHost), Password: "", }), + defaultExpiration: DefaultExpiration, } } +func (r *Redis) SetDefaultExpiration(expiration time.Duration) { + r.defaultExpiration = expiration +} + func (r *Redis) Get(ctx context.Context, key string) ([]byte, error) { return r.client.Get(ctx, key).Bytes() } func (r *Redis) Set(ctx context.Context, key string, value []byte) error { - return r.client.Set(ctx, key, value, DefaultExpiration).Err() + return r.client.Set(ctx, key, value, r.defaultExpiration).Err() } func (r *Redis) SetWithExpiration(ctx context.Context, key string, value []byte, expiration time.Duration) error { diff --git a/go.mod b/go.mod index 0d95dc5..a048cfe 100644 --- a/go.mod +++ b/go.mod @@ -21,4 +21,5 @@ require ( github.com/PuerkitoBio/goquery v1.9.1 github.com/hbollon/go-edlib v1.6.0 github.com/prometheus/client_golang v1.19.0 + github.com/xhit/go-str2duration/v2 v2.1.0 ) diff --git a/go.sum b/go.sum index 1ea89d1..7253488 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGK github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= diff --git a/main.go b/main.go index 44b0385..18467e6 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,8 @@ import ( "github.com/felipemarinho97/torrent-indexer/monitoring" "github.com/felipemarinho97/torrent-indexer/requester" "github.com/prometheus/client_golang/prometheus/promhttp" + + str2duration "github.com/xhit/go-str2duration/v2" ) func main() { @@ -19,6 +21,21 @@ func main() { flaresolverr := requester.NewFlareSolverr(os.Getenv("FLARESOLVERR_ADDRESS"), 60000) req := requester.NewRequester(flaresolverr, redis) + + // get shot-lived and long-lived cache expiration from env + shortLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("SHORT_LIVED_CACHE_EXPIRATION")) + if err == nil { + fmt.Printf("Setting short-lived cache expiration to %s\n", shortLivedCacheExpiration) + req.SetShortLivedCacheExpiration(shortLivedCacheExpiration) + } + longLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("LONG_LIVED_CACHE_EXPIRATION")) + if err == nil { + fmt.Printf("Setting long-lived cache expiration to %s\n", longLivedCacheExpiration) + redis.SetDefaultExpiration(longLivedCacheExpiration) + } else { + fmt.Println(err) + } + indexers := handler.NewIndexers(redis, metrics, req) indexerMux := http.NewServeMux() @@ -38,8 +55,14 @@ func main() { panic(err) } }() - fmt.Println("Server listening on :7006") - err := http.ListenAndServe(":7006", indexerMux) + + port := os.Getenv("PORT") + if port == "" { + port = "7006" + } + + fmt.Printf("Server listening on :%s\n", port) + err = http.ListenAndServe(":"+port, indexerMux) if err != nil { panic(err) } diff --git a/requester/requester.go b/requester/requester.go index f08ea2a..c898009 100644 --- a/requester/requester.go +++ b/requester/requester.go @@ -13,20 +13,24 @@ import ( ) const ( - shortLivedCacheExpiration = 30 * time.Minute - cacheKey = "shortLivedCache" + cacheKey = "shortLivedCache" ) var challangeRegex = regexp.MustCompile(`(?i)(just a moment|cf-chl-bypass|under attack)`) type Requster struct { - fs *FlareSolverr - c *cache.Redis - httpClient *http.Client + fs *FlareSolverr + c *cache.Redis + httpClient *http.Client + shortLivedCacheExpiration time.Duration } func NewRequester(fs *FlareSolverr, c *cache.Redis) *Requster { - return &Requster{fs: fs, httpClient: &http.Client{}, c: c} + return &Requster{fs: fs, httpClient: &http.Client{}, c: c, shortLivedCacheExpiration: 30 * time.Minute} +} + +func (i *Requster) SetShortLivedCacheExpiration(expiration time.Duration) { + i.shortLivedCacheExpiration = expiration } func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser, error) { @@ -75,7 +79,7 @@ func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser, // save response to cache if it's not a challange and body is not empty if !hasChallange(bodyByte) && len(bodyByte) > 0 { - err = i.c.SetWithExpiration(ctx, key, bodyByte, shortLivedCacheExpiration) + err = i.c.SetWithExpiration(ctx, key, bodyByte, i.shortLivedCacheExpiration) if err != nil { fmt.Printf("failed to save response to cache: %v\n", err) }