Feat/ttl config (#23)

* new: feat: add option to customize ttl

* chg: docs: add docs form ENVs

README.md

@@ -23,6 +23,19 @@ docker-compose up -d
 
 The server will be available at [http://localhost:8080/](http://localhost:8080/).
 
+## Configuration
+
+You can configure the server using the following environment variables:
+  
+- `PORT`: (optional) The port that the server will listen to. Default: `7006`
+- `FLARESOLVERR_ADDRESS`: (optional) The address of the FlareSolverr instance. Default: `N/A`
+- `REDIS_HOST`: (optional) The address of the Redis instance. Default: `localhost`
+- `SHORT_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the short-lived cache in duration format. Default: `30m`
+    - This cache is used to cache homepage or search results.
+    - Example: `30m`, `1h`, `1h30m`, `1h30m30s`
+- `LONG_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the long-lived cache in duration format. Default: `7d`
+    - This cache is used to store the torrent webpages (posts). You can set it to a higher value because the torrent pages are not updated frequently.
+
 ## Integrating with Jackett
 
 You can integrate this indexer with Jackett by adding a new Torznab custom indexer. Here is an example of how to do it for the `bludv` indexer:

cache/redis.go

@@ -9,13 +9,14 @@ import (
 	"github.com/redis/go-redis/v9"
 )
 
-var (
+const (
 	DefaultExpiration      = 24 * time.Hour * 7 // 7 days
 	IndexerComandoTorrents = "indexer:comando_torrents"
 )
 
 type Redis struct {
-	client *redis.Client
+	client            *redis.Client
+	defaultExpiration time.Duration
 }
 
 func NewRedis() *Redis {
@@ -28,15 +29,20 @@ func NewRedis() *Redis {
 			Addr:     fmt.Sprintf("%s:6379", redisHost),
 			Password: "",
 		}),
+		defaultExpiration: DefaultExpiration,
 	}
 }
 
+func (r *Redis) SetDefaultExpiration(expiration time.Duration) {
+	r.defaultExpiration = expiration
+}
+
 func (r *Redis) Get(ctx context.Context, key string) ([]byte, error) {
 	return r.client.Get(ctx, key).Bytes()
 }
 
 func (r *Redis) Set(ctx context.Context, key string, value []byte) error {
-	return r.client.Set(ctx, key, value, DefaultExpiration).Err()
+	return r.client.Set(ctx, key, value, r.defaultExpiration).Err()
 }
 
 func (r *Redis) SetWithExpiration(ctx context.Context, key string, value []byte, expiration time.Duration) error {

go.mod

@@ -21,4 +21,5 @@ require (
 	github.com/PuerkitoBio/goquery v1.9.1
 	github.com/hbollon/go-edlib v1.6.0
 	github.com/prometheus/client_golang v1.19.0
+	github.com/xhit/go-str2duration/v2 v2.1.0
 )

go.sum

@@ -28,6 +28,8 @@ github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGK
 github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
 github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
 github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
+github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

main.go

@@ -10,6 +10,8 @@ import (
 	"github.com/felipemarinho97/torrent-indexer/monitoring"
 	"github.com/felipemarinho97/torrent-indexer/requester"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	str2duration "github.com/xhit/go-str2duration/v2"
 )
 
 func main() {
@@ -19,6 +21,21 @@ func main() {
 
 	flaresolverr := requester.NewFlareSolverr(os.Getenv("FLARESOLVERR_ADDRESS"), 60000)
 	req := requester.NewRequester(flaresolverr, redis)
+
+	// get shot-lived and long-lived cache expiration from env
+	shortLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("SHORT_LIVED_CACHE_EXPIRATION"))
+	if err == nil {
+		fmt.Printf("Setting short-lived cache expiration to %s\n", shortLivedCacheExpiration)
+		req.SetShortLivedCacheExpiration(shortLivedCacheExpiration)
+	}
+	longLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("LONG_LIVED_CACHE_EXPIRATION"))
+	if err == nil {
+		fmt.Printf("Setting long-lived cache expiration to %s\n", longLivedCacheExpiration)
+		redis.SetDefaultExpiration(longLivedCacheExpiration)
+	} else {
+		fmt.Println(err)
+	}
+
 	indexers := handler.NewIndexers(redis, metrics, req)
 
 	indexerMux := http.NewServeMux()
@@ -38,8 +55,14 @@ func main() {
 			panic(err)
 		}
 	}()
-	fmt.Println("Server listening on :7006")
+
-	err := http.ListenAndServe(":7006", indexerMux)
+	port := os.Getenv("PORT")
+	if port == "" {
+		port = "7006"
+	}
+
+	fmt.Printf("Server listening on :%s\n", port)
+	err = http.ListenAndServe(":"+port, indexerMux)
 	if err != nil {
 		panic(err)
 	}

requester/requester.go

@@ -13,20 +13,24 @@ import (
 )
 
 const (
-	shortLivedCacheExpiration = 30 * time.Minute
+	cacheKey = "shortLivedCache"
-	cacheKey                  = "shortLivedCache"
 )
 
 var challangeRegex = regexp.MustCompile(`(?i)(just a moment|cf-chl-bypass|under attack)`)
 
 type Requster struct {
-	fs         *FlareSolverr
+	fs                        *FlareSolverr
-	c          *cache.Redis
+	c                         *cache.Redis
-	httpClient *http.Client
+	httpClient                *http.Client
+	shortLivedCacheExpiration time.Duration
 }
 
 func NewRequester(fs *FlareSolverr, c *cache.Redis) *Requster {
-	return &Requster{fs: fs, httpClient: &http.Client{}, c: c}
+	return &Requster{fs: fs, httpClient: &http.Client{}, c: c, shortLivedCacheExpiration: 30 * time.Minute}
+}
+
+func (i *Requster) SetShortLivedCacheExpiration(expiration time.Duration) {
+	i.shortLivedCacheExpiration = expiration
 }
 
 func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser, error) {
@@ -75,7 +79,7 @@ func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser,
 
 	// save response to cache if it's not a challange and body is not empty
 	if !hasChallange(bodyByte) && len(bodyByte) > 0 {
-		err = i.c.SetWithExpiration(ctx, key, bodyByte, shortLivedCacheExpiration)
+		err = i.c.SetWithExpiration(ctx, key, bodyByte, i.shortLivedCacheExpiration)
 		if err != nil {
 			fmt.Printf("failed to save response to cache: %v\n", err)
 		}