darklyn/torrent-indexer
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Feat/Search support (#25)

Browse Source 
* new: feat: add search support with meilisearch

* new: feat: add search interface

* new: feat: add new audio mappings

* chg: fix: add meilisearch docs

* chg: fix: lint issues

* chg: feat: add br flag

* chg: fix: use the same user agent

* chg: fix: bludv (again)

* chg: fix: lint issue
This commit is contained in:
Felipe Marinho
2024-12-13 11:54:55 -03:00
committed by GitHub
parent 0a702d1893
commit 88d6d506bf
16 changed files with 478 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -29,6 +29,8 @@ You can configure the server using the following environment variables:
- `PORT`: (optional) The port that the server will listen to. Default: `7006`
- `FLARESOLVERR_ADDRESS`: (optional) The address of the FlareSolverr instance. Default: `N/A`
- `MEILISEARCH_ADDRESS`: (optional) The address of the MeiliSearch instance. Default: `N/A`
- `MEILISEARCH_KEY`: (optional) The API key of the MeiliSearch instance. Default: `N/A`
- `REDIS_HOST`: (optional) The address of the Redis instance. Default: `localhost`
- `SHORT_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the short-lived cache in duration format. Default: `30m`
- This cache is used to cache homepage or search results.

27
api/bludv.go
View File

@@ -40,10 +40,10 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
// URL encode query param
q = url.QueryEscape(q)
url := bludv.URL
if q != "" {
url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q)
} else if page != "" {
if page != "" {
url = fmt.Sprintf("%spage/%s", url, page)
} else {
url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q)
}
fmt.Println("URL:>", url)
@@ -78,9 +78,9 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
links = append(links, link)
})
var itChan = make(chan []IndexedTorrent)
var itChan = make(chan []schema.IndexedTorrent)
var errChan = make(chan error)
indexedTorrents := []IndexedTorrent{}
indexedTorrents := []schema.IndexedTorrent{}
for _, link := range links {
go func(link string) {
torrents, err := getTorrentsBluDV(ctx, i, link)
@@ -110,16 +110,21 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
// remove the ones with zero similarity
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
return it.Similarity > 0
})
}
// sort by similarity
slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
return int((j.Similarity - i.Similarity) * 1000)
})
// send to search index
go func() {
_ = i.search.IndexTorrents(indexedTorrents)
}()
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(Response{
Results: indexedTorrents,
@@ -130,8 +135,8 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
}
}
func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
var indexedTorrents []IndexedTorrent
func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
var indexedTorrents []schema.IndexedTorrent
doc, err := getDocument(ctx, i, link)
if err != nil {
return nil, err
@@ -191,7 +196,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
size = stableUniq(size)
var chanIndexedTorrent = make(chan IndexedTorrent)
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
// for each magnet link, create a new indexed torrent
for it, magnetLink := range magnetLinks {
@@ -231,7 +236,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
mySize = size[it]
}
ixt := IndexedTorrent{
ixt := schema.IndexedTorrent{
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
OriginalTitle: title,
Details: link,

21
api/comando_torrents.go
View File

@@ -93,9 +93,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
links = append(links, link)
})
var itChan = make(chan []IndexedTorrent)
var itChan = make(chan []schema.IndexedTorrent)
var errChan = make(chan error)
indexedTorrents := []IndexedTorrent{}
indexedTorrents := []schema.IndexedTorrent{}
for _, link := range links {
go func(link string) {
torrents, err := getTorrents(ctx, i, link)
@@ -125,16 +125,21 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
// remove the ones with zero similarity
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
return it.Similarity > 0
})
}
// sort by similarity
slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
return int((j.Similarity - i.Similarity) * 1000)
})
// send to search index
go func() {
_ = i.search.IndexTorrents(indexedTorrents)
}()
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(Response{
Results: indexedTorrents,
@@ -145,8 +150,8 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
}
}
func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
var indexedTorrents []IndexedTorrent
func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
var indexedTorrents []schema.IndexedTorrent
doc, err := getDocument(ctx, i, link)
if err != nil {
return nil, err
@@ -221,7 +226,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
size = stableUniq(size)
var chanIndexedTorrent = make(chan IndexedTorrent)
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
// for each magnet link, create a new indexed torrent
for it, magnetLink := range magnetLinks {
@@ -261,7 +266,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
mySize = size[it]
}
ixt := IndexedTorrent{
ixt := schema.IndexedTorrent{
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
OriginalTitle: title,
Details: link,

35
api/index.go
View File

@@ -9,12 +9,14 @@ import (
"github.com/felipemarinho97/torrent-indexer/monitoring"
"github.com/felipemarinho97/torrent-indexer/requester"
"github.com/felipemarinho97/torrent-indexer/schema"
meilisearch "github.com/felipemarinho97/torrent-indexer/search"
)
type Indexer struct {
redis *cache.Redis
metrics *monitoring.Metrics
requester *requester.Requster
search *meilisearch.SearchIndexer
}
type IndexerMeta struct {
@@ -23,32 +25,16 @@ type IndexerMeta struct {
}
type Response struct {
Results []IndexedTorrent `json:"results"`
Count int `json:"count"`
Results []schema.IndexedTorrent `json:"results"`
Count int `json:"count"`
}
type IndexedTorrent struct {
Title string `json:"title"`
OriginalTitle string `json:"original_title"`
Details string `json:"details"`
Year string `json:"year"`
IMDB string `json:"imdb"`
Audio []schema.Audio `json:"audio"`
MagnetLink string `json:"magnet_link"`
Date time.Time `json:"date"`
InfoHash string `json:"info_hash"`
Trackers []string `json:"trackers"`
Size string `json:"size"`
LeechCount int `json:"leech_count"`
SeedCount int `json:"seed_count"`
Similarity float32 `json:"similarity"`
}
func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster) *Indexer {
func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster, si *meilisearch.SearchIndexer) *Indexer {
return &Indexer{
redis: redis,
metrics: metrics,
requester: req,
search: si,
}
}
@@ -103,6 +89,15 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
"description": "Get all manual torrents",
},
},
"/search": []map[string]interface{}{
{
"method": "GET",
"description": "Search for cached torrents across all indexers",
"query_params": map[string]string{
"q": "search query",
},
},
},
},
})
if err != nil {

4
api/manual.go
View File

@@ -26,7 +26,7 @@ type ManualIndexerRequest struct {
func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var req ManualIndexerRequest
indexedTorrents := []IndexedTorrent{}
indexedTorrents := []schema.IndexedTorrent{}
// fetch from redis
out, err := i.redis.Get(ctx, manualTorrentsRedisKey)
@@ -97,7 +97,7 @@ func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
title := processTitle(releaseTitle, magnetAudio)
ixt := IndexedTorrent{
ixt := schema.IndexedTorrent{
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
OriginalTitle: title,
Audio: magnetAudio,

55
api/search.go Normal file
View File

@@ -0,0 +1,55 @@
package handler
import (
"encoding/json"
"net/http"
"strconv"
meilisearch "github.com/felipemarinho97/torrent-indexer/search"
)
// MeilisearchHandler handles HTTP requests for Meilisearch integration.
type MeilisearchHandler struct {
Module *meilisearch.SearchIndexer
}
// NewMeilisearchHandler creates a new instance of MeilisearchHandler.
func NewMeilisearchHandler(module *meilisearch.SearchIndexer) *MeilisearchHandler {
return &MeilisearchHandler{Module: module}
}
// SearchTorrentHandler handles the searching of torrent items.
func (h *MeilisearchHandler) SearchTorrentHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
query := r.URL.Query().Get("q")
if query == "" {
http.Error(w, "Query parameter 'q' is required", http.StatusBadRequest)
return
}
limitStr := r.URL.Query().Get("limit")
limit := 10 // Default limit
if limitStr != "" {
var err error
limit, err = strconv.Atoi(limitStr)
if err != nil || limit <= 0 {
http.Error(w, "Invalid limit parameter", http.StatusBadRequest)
return
}
}
results, err := h.Module.SearchTorrent(query, limit)
if err != nil {
http.Error(w, "Failed to search torrents", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(results); err != nil {
http.Error(w, "Failed to encode response", http.StatusInternalServerError)
}
}

21
api/torrent_dos_filmes.go
View File

@@ -77,9 +77,9 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
links = append(links, link)
})
var itChan = make(chan []IndexedTorrent)
var itChan = make(chan []schema.IndexedTorrent)
var errChan = make(chan error)
indexedTorrents := []IndexedTorrent{}
indexedTorrents := []schema.IndexedTorrent{}
for _, link := range links {
go func(link string) {
torrents, err := getTorrentsTorrentDosFilmes(ctx, i, link)
@@ -109,16 +109,21 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
// remove the ones with zero similarity
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
indexedTorrents = utils.Filter(indexedTorrents, func(it IndexedTorrent) bool {
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
return it.Similarity > 0
})
}
// sort by similarity
slices.SortFunc(indexedTorrents, func(i, j IndexedTorrent) int {
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
return int((j.Similarity - i.Similarity) * 1000)
})
// send to search index
go func() {
_ = i.search.IndexTorrents(indexedTorrents)
}()
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(Response{
Results: indexedTorrents,
@@ -129,8 +134,8 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
}
}
func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
var indexedTorrents []IndexedTorrent
func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
var indexedTorrents []schema.IndexedTorrent
doc, err := getDocument(ctx, i, link)
if err != nil {
return nil, err
@@ -190,7 +195,7 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
size = stableUniq(size)
var chanIndexedTorrent = make(chan IndexedTorrent)
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
// for each magnet link, create a new indexed torrent
for it, magnetLink := range magnetLinks {
@@ -230,7 +235,7 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
mySize = size[it]
}
ixt := IndexedTorrent{
ixt := schema.IndexedTorrent{
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
OriginalTitle: title,
Details: link,

14
docker-compose.yml
View File

@@ -11,6 +11,8 @@ services:
- indexer
environment:
- REDIS_HOST=redis
- MEILISEARCH_ADDRESS=http://meilisearch:7700
- MEILISEARCH_KEY=my-secret-key
- FLARESOLVERR_ADDRESS=http://flaresolverr:8191
redis:
@@ -20,5 +22,17 @@ services:
networks:
- indexer
# This container is not necessary for the indexer to work,
# deploy if you want to use the search feature
meilisearch:
image: getmeili/meilisearch:latest
container_name: meilisearch
restart: unless-stopped
networks:
- indexer
environment:
- MEILI_NO_ANALYTICS=true
- MEILI_MASTER_KEY=my-secret-key
networks:
indexer:

8
main.go
View File

@@ -8,7 +8,9 @@ import (
handler "github.com/felipemarinho97/torrent-indexer/api"
"github.com/felipemarinho97/torrent-indexer/cache"
"github.com/felipemarinho97/torrent-indexer/monitoring"
"github.com/felipemarinho97/torrent-indexer/public"
"github.com/felipemarinho97/torrent-indexer/requester"
meilisearch "github.com/felipemarinho97/torrent-indexer/search"
"github.com/prometheus/client_golang/prometheus/promhttp"
str2duration "github.com/xhit/go-str2duration/v2"
@@ -16,6 +18,7 @@ import (
func main() {
redis := cache.NewRedis()
searchIndex := meilisearch.NewSearchIndexer(os.Getenv("MEILISEARCH_ADDRESS"), os.Getenv("MEILISEARCH_KEY"), "torrents")
metrics := monitoring.NewMetrics()
metrics.Register()
@@ -36,7 +39,8 @@ func main() {
fmt.Println(err)
}
indexers := handler.NewIndexers(redis, metrics, req)
indexers := handler.NewIndexers(redis, metrics, req, searchIndex)
search := handler.NewMeilisearchHandler(searchIndex)
indexerMux := http.NewServeMux()
metricsMux := http.NewServeMux()
@@ -46,6 +50,8 @@ func main() {
indexerMux.HandleFunc("/indexers/torrent-dos-filmes", indexers.HandlerTorrentDosFilmesIndexer)
indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer)
indexerMux.HandleFunc("/search", search.SearchTorrentHandler)
indexerMux.Handle("/ui/", http.StripPrefix("/ui/", http.FileServer(http.FS(public.UIFiles))))
metricsMux.Handle("/metrics", promhttp.Handler())

6
public/index.go Normal file
View File

@@ -0,0 +1,6 @@
package public
import "embed"
//go:embed *
var UIFiles embed.FS

127
public/index.html Normal file
View File

@@ -0,0 +1,127 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Torrent Indexer</title>
<script src="https://cdn.tailwindcss.com"></script>
<script src="https://cdn.jsdelivr.net/npm/@heroicons/react/solid@2.0.0/dist/index.umd.js"></script>
</head>
<body class="bg-gray-900 text-white font-sans">
<div class="container mx-auto p-6">
<!-- Header -->
<header class="text-center mb-10">
<h1 class="text-4xl font-bold text-blue-400">Torrent Indexer 🇧🇷</h1>
<p class="text-gray-400 mt-2">Find torrents with detailed information from torrent-indexer cache</p>
</header>
<!-- Search Bar -->
<div class="flex justify-center mb-10">
<input id="search-query" type="text" placeholder="Enter search query"
class="w-full max-w-lg px-4 py-2 rounded-md border border-gray-600 bg-gray-800 text-white focus:ring focus:ring-blue-500">
<button id="search-btn"
class="ml-4 px-6 py-2 bg-blue-600 hover:bg-blue-700 rounded-md font-bold text-white">Search</button>
</div>
<!-- Results Section -->
<div id="results" class="space-y-6">
<!-- Dynamic content will be injected here -->
</div>
</div>
<script>
// Function to render a single torrent result
function renderTorrent(torrent) {
return `
<div class="p-6 bg-gray-800 rounded-lg shadow-md flex flex-col md:flex-row gap-6">
<!-- Torrent Title and Details -->
<div class="flex-grow">
<h2 class="text-2xl font-bold text-blue-400 flex items-center gap-2">
<span>${torrent.title}</span>
<span class="text-sm text-gray-400">(${torrent.year})</span>
</h2>
<p class="text-gray-500 italic mt-1">${torrent.original_title}</p>
<div class="mt-4 grid grid-cols-1 sm:grid-cols-2 gap-4">
<p><strong>Audio:</strong> ${torrent.audio.join(', ')}</p>
<p><strong>Size:</strong> ${torrent.size}</p>
<p><strong>Seeds:</strong> ${torrent.seed_count} | <strong>Leeches:</strong> ${torrent.leech_count}</p>
<p><strong>Info Hash:</strong> <span class="text-sm break-all text-gray-300">${torrent.info_hash}</span></p>
</div>
</div>
<!-- Actions -->
<div class="flex flex-col justify-between items-start md:items-end">
<div>
<a href="${torrent.imdb}" target="_blank"
class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium">
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M8 16l-4-4m0 0l4-4m-4 4h16" />
</svg>
View on IMDB
</a>
<a href="${torrent.details}" target="_blank"
class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium mt-2">
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M13 16h-1v-4h-.01M9 20h6a2 2 0 002-2v-5a2 2 0 00-2-2h-3.5a2 2 0 00-1.85 1.19M13 10V6a3 3 0 00-6 0v4" />
</svg>
View Details
</a>
</div>
<a href="${torrent.magnet_link}" target="_blank"
class="px-4 py-2 bg-green-600 hover:bg-green-700 text-white font-bold rounded-md flex items-center gap-2 mt-4">
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M9 17v-6m6 6v-6m-6 6l-2-2m8 0l2-2M5 9l7-7 7 7" />
</svg>
Download Magnet
</a>
</div>
</div>
`;
}
// Handle search
async function onSearch() {
const query = document.getElementById('search-query').value.trim();
if (!query) {
alert('Please enter a search query!');
return;
}
try {
const response = await fetch(`/search?q=${encodeURIComponent(query)}`);
if (!response.ok) {
throw new Error('Search failed');
}
const results = await response.json();
const resultsContainer = document.getElementById('results');
resultsContainer.innerHTML = results.map(renderTorrent).join('');
} catch (error) {
// add error element
document.getElementById('results').innerHTML = `
<div class="p-6 bg-red-800 rounded-lg shadow-md text-center">
<p class="text-xl font-bold text-red-400">Error fetching search results</p>
<p class="text-gray-400 mt-2">Please try again later.</p>
</div>
`;
//alert('Error fetching search results. Please try again.');
console.error(error);
}
}
document.getElementById('search-btn').addEventListener('click', onSearch);
// on enter press
document.getElementById('search-query').addEventListener('keypress', (e) => {
if (e.key === 'Enter') {
onSearch();
}
});
</script>
</body>
</html>

20
requester/flaresolverr.go
View File

@@ -10,6 +10,8 @@ import (
"net/url"
"strings"
"sync"
"github.com/felipemarinho97/torrent-indexer/utils"
)
type FlareSolverr struct {
@@ -243,8 +245,15 @@ func (f *FlareSolverr) Get(_url string) (io.ReadCloser, error) {
return nil, fmt.Errorf("under attack")
}
// check if the response is valid HTML
if !utils.IsValidHTML(response.Solution.Response) {
fmt.Printf("[FlareSolverr] Invalid HTML response from %s\n", _url)
response.Solution.Response = ""
}
// If the response body is empty but cookies are present, make a new request
if response.Solution.Response == "" && len(response.Solution.Cookies) > 0 {
fmt.Printf("[FlareSolverr] Making a new request to %s with cookies\n", _url)
// Create a new request with cookies
client := &http.Client{}
cookieJar, err := cookiejar.New(&cookiejar.Options{})
@@ -268,13 +277,22 @@ func (f *FlareSolverr) Get(_url string) (io.ReadCloser, error) {
return nil, err
}
// use the same user returned by the FlareSolverr
secondReq.Header.Set("User-Agent", response.Solution.UserAgent)
secondResp, err := client.Do(secondReq)
if err != nil {
return nil, err
}
respByte := new(bytes.Buffer)
_, err = respByte.ReadFrom(secondResp.Body)
if err != nil {
return nil, err
}
// Return the body of the second request
return secondResp.Body, nil
return io.NopCloser(bytes.NewReader(respByte.Bytes())), nil
}
// Return the original response body

2
schema/audio.go
View File

@@ -33,6 +33,8 @@ const (
AudioThai2 = "Tailandes"
AudioTurkish = "Turco"
AudioHindi = "Hindi"
AudioFarsi = "Persa"
AudioMalay = "Malaio"
)
var AudioList = []Audio{

20
schema/indexed_torrent.go Normal file
View File

@@ -0,0 +1,20 @@
package schema
import "time"
type IndexedTorrent struct {
Title string `json:"title"`
OriginalTitle string `json:"original_title"`
Details string `json:"details"`
Year string `json:"year"`
IMDB string `json:"imdb"`
Audio []Audio `json:"audio"`
MagnetLink string `json:"magnet_link"`
Date time.Time `json:"date"`
InfoHash string `json:"info_hash"`
Trackers []string `json:"trackers"`
Size string `json:"size"`
LeechCount int `json:"leech_count"`
SeedCount int `json:"seed_count"`
Similarity float32 `json:"similarity"`
}

147
search/meilisearch.go Normal file
View File

@@ -0,0 +1,147 @@
package meilisearch
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"github.com/felipemarinho97/torrent-indexer/schema"
)
// SearchIndexer integrates with Meilisearch to index and search torrent items.
type SearchIndexer struct {
Client *http.Client
BaseURL string
APIKey string
IndexName string
}
// NewSearchIndexer creates a new instance of SearchIndexer.
func NewSearchIndexer(baseURL, apiKey, indexName string) *SearchIndexer {
return &SearchIndexer{
Client: &http.Client{Timeout: 10 * time.Second},
BaseURL: baseURL,
APIKey: apiKey,
IndexName: indexName,
}
}
// IndexTorrent indexes a single torrent item in Meilisearch.
func (t *SearchIndexer) IndexTorrent(torrent schema.IndexedTorrent) error {
url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
torrentWithKey := struct {
Hash string `json:"id"`
schema.IndexedTorrent
}{
Hash: torrent.InfoHash,
IndexedTorrent: torrent,
}
jsonData, err := json.Marshal(torrentWithKey)
if err != nil {
return fmt.Errorf("failed to marshal torrent data: %w", err)
}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
if t.APIKey != "" {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
}
resp, err := t.Client.Do(req)
if err != nil {
return fmt.Errorf("failed to execute request: %w", err)
}
defer resp.Body.Close()
return nil
}
func (t *SearchIndexer) IndexTorrents(torrents []schema.IndexedTorrent) error {
url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
torrentsWithKey := make([]struct {
Hash string `json:"id"`
schema.IndexedTorrent
}, 0, len(torrents))
for _, torrent := range torrents {
torrentWithKey := struct {
Hash string `json:"id"`
schema.IndexedTorrent
}{
Hash: torrent.InfoHash,
IndexedTorrent: torrent,
}
torrentsWithKey = append(torrentsWithKey, torrentWithKey)
}
jsonData, err := json.Marshal(torrentsWithKey)
if err != nil {
return fmt.Errorf("failed to marshal torrent data: %w", err)
}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
if t.APIKey != "" {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
}
resp, err := t.Client.Do(req)
if err != nil {
return fmt.Errorf("failed to execute request: %w", err)
}
defer resp.Body.Close()
return nil
}
// SearchTorrent searches indexed torrents in Meilisearch based on the query.
func (t *SearchIndexer) SearchTorrent(query string, limit int) ([]schema.IndexedTorrent, error) {
url := fmt.Sprintf("%s/indexes/%s/search", t.BaseURL, t.IndexName)
requestBody := map[string]string{
"q": query,
}
jsonData, err := json.Marshal(requestBody)
if err != nil {
return nil, fmt.Errorf("failed to marshal search query: %w", err)
}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
if t.APIKey != "" {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
}
resp, err := t.Client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to execute request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("search failed: %s", body)
}
var result struct {
Hits []schema.IndexedTorrent `json:"hits"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, fmt.Errorf("failed to parse search response: %w", err)
}
return result.Hits, nil
}

20
utils/util.go
View File

@@ -1,5 +1,7 @@
package utils
import "regexp"
func Filter[A any](arr []A, f func(A) bool) []A {
var res []A
res = make([]A, 0)
@@ -10,3 +12,21 @@ func Filter[A any](arr []A, f func(A) bool) []A {
}
return res
}
func IsValidHTML(input string) bool {
// Check for <!DOCTYPE> declaration (case-insensitive)
doctypeRegex := regexp.MustCompile(`(?i)<!DOCTYPE\s+html>`)
if !doctypeRegex.MatchString(input) {
return false
}
// Check for <html> and </html> tags (case-insensitive)
htmlTagRegex := regexp.MustCompile(`(?i)<html[\s\S]*?>[\s\S]*?</html>`)
if !htmlTagRegex.MatchString(input) {
return false
}
// Check for <body> and </body> tags (case-insensitive)
bodyTagRegex := regexp.MustCompile(`(?i)<body[\s\S]*?>[\s\S]*?</body>`)
return bodyTagRegex.MatchString(input)
}