From d9141c8df70bac04341e3e00418c5f40dabb09be Mon Sep 17 00:00:00 2001 From: Felipe Marinho Date: Tue, 29 Jul 2025 12:34:37 -0300 Subject: [PATCH] new: feat: add magnet-metadata-api post processor (#39) * new: feat: add magnet-metadata-api post processor * chg: fix: lint issue * chg: chore: comment optional containers * chg: fix: remove redundant check --- api/bludv.go | 9 ++- api/comando_torrents.go | 7 ++- api/comandohds.go | 7 ++- api/index.go | 40 +++++++----- api/post_processors.go | 39 ++++++++++++ api/rede_torrent.go | 7 ++- api/starck_filmes.go | 7 ++- api/torrent_dos_filmes.go | 9 ++- docker-compose.yml | 61 ++++++++++++++---- magnet/magnet_metadata_api.go | 113 ++++++++++++++++++++++++++++++++++ main.go | 15 ++++- schema/indexed_torrent.go | 6 ++ scrape/info.go | 90 ++++++++++++++++++++++++++- utils/util.go | 20 +++++- 14 files changed, 389 insertions(+), 41 deletions(-) create mode 100644 magnet/magnet_metadata_api.go diff --git a/api/bludv.go b/api/bludv.go index 6eb42c7..a1ca367 100644 --- a/api/bludv.go +++ b/api/bludv.go @@ -80,7 +80,7 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) { }) // extract each torrent link - indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) { + indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) { return getTorrentsBluDV(ctx, i, link) }) @@ -137,7 +137,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In // if decoded magnet link is indeed a magnet link, append it if strings.HasPrefix(magnetLinkDecoded, "magnet:") { magnetLinks = append(magnetLinks, magnetLinkDecoded) - } else { + } else if !strings.Contains(magnetLinkDecoded, "watch.brplayer") { fmt.Printf("WARN: link \"%s\" decoding resulted in non-magnet link: %s\n", href, magnetLinkDecoded) } }) @@ -211,6 +211,11 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In if len(size) == len(magnetLinks) { mySize = size[it] } + if mySize == "" { + go func() { + _, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink) + }() + } ixt := schema.IndexedTorrent{ Title: releaseTitle, diff --git a/api/comando_torrents.go b/api/comando_torrents.go index c04ffc2..cfae586 100644 --- a/api/comando_torrents.go +++ b/api/comando_torrents.go @@ -94,7 +94,7 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) }) // extract each torrent link - indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) { + indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) { return getTorrents(ctx, i, link) }) @@ -208,6 +208,11 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.Indexed if len(size) == len(magnetLinks) { mySize = size[it] } + if mySize == "" { + go func() { + _, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink) + }() + } ixt := schema.IndexedTorrent{ Title: releaseTitle, diff --git a/api/comandohds.go b/api/comandohds.go index e88739b..f20a669 100644 --- a/api/comandohds.go +++ b/api/comandohds.go @@ -82,7 +82,7 @@ func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Reques }) // extract each torrent link - indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) { + indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) { return getTorrentsComandoHDs(ctx, i, link) }) @@ -192,6 +192,11 @@ func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]sche if len(size) == len(magnetLinks) { mySize = size[it] } + if mySize == "" { + go func() { + _, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink) + }() + } ixt := schema.IndexedTorrent{ Title: releaseTitle, diff --git a/api/index.go b/api/index.go index 0964b88..af509b8 100644 --- a/api/index.go +++ b/api/index.go @@ -6,6 +6,7 @@ import ( "time" "github.com/felipemarinho97/torrent-indexer/cache" + "github.com/felipemarinho97/torrent-indexer/magnet" "github.com/felipemarinho97/torrent-indexer/monitoring" "github.com/felipemarinho97/torrent-indexer/requester" "github.com/felipemarinho97/torrent-indexer/schema" @@ -13,11 +14,12 @@ import ( ) type Indexer struct { - redis *cache.Redis - metrics *monitoring.Metrics - requester *requester.Requster - search *meilisearch.SearchIndexer - postProcessors []PostProcessorFunc + redis *cache.Redis + metrics *monitoring.Metrics + requester *requester.Requster + search *meilisearch.SearchIndexer + magnetMetadataAPI *magnet.MetadataClient + postProcessors []PostProcessorFunc } type IndexerMeta struct { @@ -35,19 +37,27 @@ type Response struct { type PostProcessorFunc func(*Indexer, *http.Request, []schema.IndexedTorrent) []schema.IndexedTorrent var GlobalPostProcessors = []PostProcessorFunc{ - AddSimilarityCheck, // Jaccard similarity - CleanupTitleWebsites, // Remove website names from titles - AppendAudioTags, // Add (brazilian, eng, etc.) audio tags to titles - SendToSearchIndexer, // Send indexed torrents to Meilisearch + AddSimilarityCheck, // Jaccard similarity + FullfilMissingMetadata, // Fill missing size or title metadata + CleanupTitleWebsites, // Remove website names from titles + AppendAudioTags, // Add (brazilian, eng, etc.) audio tags to titles + SendToSearchIndexer, // Send indexed torrents to Meilisearch } -func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster, si *meilisearch.SearchIndexer) *Indexer { +func NewIndexers( + redis *cache.Redis, + metrics *monitoring.Metrics, + req *requester.Requster, + si *meilisearch.SearchIndexer, + mc *magnet.MetadataClient, +) *Indexer { return &Indexer{ - redis: redis, - metrics: metrics, - requester: req, - search: si, - postProcessors: GlobalPostProcessors, + redis: redis, + metrics: metrics, + requester: req, + search: si, + magnetMetadataAPI: mc, + postProcessors: GlobalPostProcessors, } } diff --git a/api/post_processors.go b/api/post_processors.go index dd68acc..0a92f9b 100644 --- a/api/post_processors.go +++ b/api/post_processors.go @@ -35,6 +35,45 @@ func SendToSearchIndexer(i *Indexer, _ *http.Request, torrents []schema.IndexedT return torrents } +// FullfilMissingMetadata fills in missing metadata for indexed torrents +func FullfilMissingMetadata(i *Indexer, r *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent { + if !i.magnetMetadataAPI.IsEnabled() { + return torrents + } + + return utils.ParallelFlatMap(torrents, func(it schema.IndexedTorrent) ([]schema.IndexedTorrent, error) { + if it.Size != "" && it.Title != "" && it.OriginalTitle != "" { + return []schema.IndexedTorrent{it}, nil + } + m, err := i.magnetMetadataAPI.FetchMetadata(r.Context(), it.MagnetLink) + if err != nil { + return []schema.IndexedTorrent{it}, nil + } + + // convert size in bytes to a human-readable format + it.Size = utils.FormatBytes(m.Size) + + // Use name from metadata if available as it is more accurate + if m.Name != "" { + it.Title = m.Name + } + fmt.Printf("hash: %s get -> size: %s\n", m.InfoHash, it.Size) + + // If files are present, add them to the indexed torrent + if len(m.Files) > 0 { + it.Files = make([]schema.File, len(m.Files)) + for i, file := range m.Files { + it.Files[i] = schema.File{ + Path: file.Path, + Size: utils.FormatBytes(file.Size), + } + } + } + + return []schema.IndexedTorrent{it}, nil + }) +} + func AddSimilarityCheck(i *Indexer, r *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent { q := r.URL.Query().Get("q") diff --git a/api/rede_torrent.go b/api/rede_torrent.go index 7e4339c..1225742 100644 --- a/api/rede_torrent.go +++ b/api/rede_torrent.go @@ -80,7 +80,7 @@ func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Reque }) // extract each torrent link - indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) { + indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) { return getTorrentsRedeTorrent(ctx, i, link) }) @@ -216,6 +216,11 @@ func getTorrentsRedeTorrent(ctx context.Context, i *Indexer, link string) ([]sch if len(size) == len(magnetLinks) { mySize = size[it] } + if mySize == "" { + go func() { + _, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink) + }() + } ixt := schema.IndexedTorrent{ Title: releaseTitle, diff --git a/api/starck_filmes.go b/api/starck_filmes.go index 20e5698..9cc5560 100644 --- a/api/starck_filmes.go +++ b/api/starck_filmes.go @@ -79,7 +79,7 @@ func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Requ }) // extract each torrent link - indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) { + indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) { return getTorrentStarckFilmes(ctx, i, link) }) @@ -192,6 +192,11 @@ func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]sch if len(size) == len(magnetLinks) { mySize = size[it] } + if mySize == "" { + go func() { + _, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink) + }() + } ixt := schema.IndexedTorrent{ Title: releaseTitle, diff --git a/api/torrent_dos_filmes.go b/api/torrent_dos_filmes.go index a0f2f74..1c9d4fc 100644 --- a/api/torrent_dos_filmes.go +++ b/api/torrent_dos_filmes.go @@ -21,7 +21,7 @@ var torrent_dos_filmes = IndexerMeta{ Label: "torrent_dos_filmes", URL: "https://torrentdosfilmes.se/", SearchURL: "?s=", - PagePattern: "page/%s", + PagePattern: "category/dublado/page/%s", } func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.Request) { @@ -79,7 +79,7 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http. }) // extract each torrent link - indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) { + indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) { return getTorrentsTorrentDosFilmes(ctx, i, link) }) @@ -186,6 +186,11 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ( if len(size) == len(magnetLinks) { mySize = size[it] } + if mySize == "" { + go func() { + _, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink) + }() + } ixt := schema.IndexedTorrent{ Title: releaseTitle, diff --git a/docker-compose.yml b/docker-compose.yml index a24d941..f1567b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,9 +11,16 @@ services: - indexer environment: - REDIS_HOST=redis - - MEILISEARCH_ADDRESS=http://meilisearch:7700 - - MEILISEARCH_KEY=my-secret-key - FLARESOLVERR_ADDRESS=http://flaresolverr:8191 + + ## Meilisearch configuration (optional) + # - MEILISEARCH_ADDRESS=http://meilisearch:7700 + # - MEILISEARCH_KEY=my-secret-key + + ## Magnet Metadata API configuration (optional) + # - MAGNET_METADATA_API_ENABLED=false + # - MAGNET_METADATA_API_ADDRESS=http://magnet-metadata-api:8080 + # - MAGNET_METADATA_API_TIMEOUT_SECONDS=10 redis: image: redis:alpine @@ -22,17 +29,45 @@ services: networks: - indexer - # This container is not necessary for the indexer to work, - # deploy if you want to use the search feature - meilisearch: - image: getmeili/meilisearch:latest - container_name: meilisearch - restart: unless-stopped - networks: - - indexer - environment: - - MEILI_NO_ANALYTICS=true - - MEILI_MASTER_KEY=my-secret-key + ##### MEILISEARCH ##### + ## This container is not necessary for the indexer to work, + ## deploy if you want to use the search feature + # + # meilisearch: + # image: getmeili/meilisearch:latest + # container_name: meilisearch + # restart: unless-stopped + # networks: + # - indexer + # environment: + # - MEILI_NO_ANALYTICS=true + # - MEILI_MASTER_KEY=my-secret-key + + ##### MAGNET METADATA API ##### + ## This container is not necessary for the indexer to work, + ## deploy if you want to fetch metadata from p2p network + ## CAUTION: Never deploy this container on a cloud server (AWS, GCP, Azure, Oracle), or you will get banned! + # + # magnet-metadata-api: + # image: felipemarinho97/magnet-metadata-api:latest + # container_name: magnet-metadata-api + # restart: unless-stopped + # ports: + # - "8999:8080" + # - "42069:42069" + # networks: + # - indexer + # environment: + # - PORT=8080 + # - REDIS_URL=redis://redis:6379 + # - CACHE_DIR=/home/torrent/cache + # - ENABLE_DOWNLOADS=false + # - DOWNLOAD_BASE_URL=http://localhost:8999 + # - CLIENT_PORT=42069 + # - SEEDING_ENABLED=false + # - FALLBACK_INITIAL_CHUNK_SIZE_KB=24 + # volumes: + # - ./magnet-metadata-cache:/home/torrent/cache networks: indexer: diff --git a/magnet/magnet_metadata_api.go b/magnet/magnet_metadata_api.go new file mode 100644 index 0000000..62d9ff3 --- /dev/null +++ b/magnet/magnet_metadata_api.go @@ -0,0 +1,113 @@ +package magnet + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/felipemarinho97/torrent-indexer/cache" +) + +type MetadataRequest struct { + MagnetURI string `json:"magnet_uri"` +} + +type TorrentFile struct { + Path string `json:"path"` + Size int64 `json:"size"` + Offset int64 `json:"offset"` +} + +type MetadataResponse struct { + InfoHash string `json:"info_hash"` + Name string `json:"name"` + Size int64 `json:"size"` + Files []TorrentFile `json:"files"` + CreatedBy string `json:"created_by"` + CreatedAt time.Time `json:"created_at"` + Comment string `json:"comment"` + Trackers []string `json:"trackers"` + DownloadURL string `json:"download_url"` +} + +type MetadataClient struct { + baseURL string + httpClient *http.Client + c *cache.Redis +} + +func NewClient(baseURL string, timeout time.Duration, c *cache.Redis) *MetadataClient { + return &MetadataClient{ + baseURL: baseURL, + httpClient: &http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + MaxIdleConns: 100, + IdleConnTimeout: 30 * time.Second, + ForceAttemptHTTP2: true, + }, + }, + c: c, + } +} + +func (c *MetadataClient) IsEnabled() bool { + return c != nil && c.baseURL != "" +} + +func (c *MetadataClient) FetchMetadata(ctx context.Context, magnetURI string) (*MetadataResponse, error) { + if !c.IsEnabled() { + return nil, fmt.Errorf("magnet metadata API is not enabled") + } + // Check cache first + m, err := ParseMagnetUri(magnetURI) + if err != nil { + return nil, fmt.Errorf("failed to parse magnet URI: %w", err) + } + cacheKey := fmt.Sprintf("metadata:%s", m.InfoHash) + cachedData, err := c.c.Get(ctx, cacheKey) + if err == nil && cachedData != nil { + var cachedMetadata MetadataResponse + if err := json.Unmarshal(cachedData, &cachedMetadata); err == nil { + return &cachedMetadata, nil + } + } + + reqBody := MetadataRequest{MagnetURI: magnetURI} + body, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request body: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/api/v1/metadata", bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send POST request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("API responded with status: %s", resp.Status) + } + + var metadata MetadataResponse + if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + // Cache the metadata response + cacheData, err := json.Marshal(metadata) + if err == nil { + _ = c.c.SetWithExpiration(ctx, cacheKey, cacheData, 7*24*time.Hour) + } + + return &metadata, nil +} diff --git a/main.go b/main.go index ba645d9..1aa121c 100644 --- a/main.go +++ b/main.go @@ -4,9 +4,12 @@ import ( "fmt" "net/http" "os" + "strconv" + "time" handler "github.com/felipemarinho97/torrent-indexer/api" "github.com/felipemarinho97/torrent-indexer/cache" + "github.com/felipemarinho97/torrent-indexer/magnet" "github.com/felipemarinho97/torrent-indexer/monitoring" "github.com/felipemarinho97/torrent-indexer/public" "github.com/felipemarinho97/torrent-indexer/requester" @@ -19,6 +22,16 @@ import ( func main() { redis := cache.NewRedis() searchIndex := meilisearch.NewSearchIndexer(os.Getenv("MEILISEARCH_ADDRESS"), os.Getenv("MEILISEARCH_KEY"), "torrents") + var magnetMetadataAPI *magnet.MetadataClient + if os.Getenv("MAGNET_METADATA_API_ENABLED") == "true" { + timeout := 10 * time.Second + if v := os.Getenv("MAGNET_METADATA_API_TIMEOUT_SECONDS"); v != "" { + if t, err := strconv.Atoi(v); err == nil { + timeout = time.Duration(t) * time.Second + } + } + magnetMetadataAPI = magnet.NewClient(os.Getenv("MAGNET_METADATA_API_ADDRESS"), timeout, redis) + } metrics := monitoring.NewMetrics() metrics.Register() @@ -39,7 +52,7 @@ func main() { fmt.Println(err) } - indexers := handler.NewIndexers(redis, metrics, req, searchIndex) + indexers := handler.NewIndexers(redis, metrics, req, searchIndex, magnetMetadataAPI) search := handler.NewMeilisearchHandler(searchIndex) indexerMux := http.NewServeMux() diff --git a/schema/indexed_torrent.go b/schema/indexed_torrent.go index 0c1ac82..ecb35ff 100644 --- a/schema/indexed_torrent.go +++ b/schema/indexed_torrent.go @@ -14,7 +14,13 @@ type IndexedTorrent struct { InfoHash string `json:"info_hash"` Trackers []string `json:"trackers"` Size string `json:"size"` + Files []File `json:"files,omitempty"` LeechCount int `json:"leech_count"` SeedCount int `json:"seed_count"` Similarity float32 `json:"similarity"` } + +type File struct { + Path string `json:"path"` + Size string `json:"size"` +} diff --git a/scrape/info.go b/scrape/info.go index b01e031..2b82bfa 100644 --- a/scrape/info.go +++ b/scrape/info.go @@ -8,6 +8,7 @@ import ( "github.com/felipemarinho97/torrent-indexer/cache" "github.com/felipemarinho97/torrent-indexer/monitoring" + "github.com/felipemarinho97/torrent-indexer/utils" ) type peers struct { @@ -45,6 +46,86 @@ func setPeersToCache(ctx context.Context, r *cache.Redis, infoHash string, peer, return nil } +var additionalTrackers = []string{ + "udp://tracker.opentrackr.org:1337/announce", + "udp://p4p.arenabg.com:1337/announce", + "udp://retracker.hotplug.ru:2710/announce", + "http://tracker.bt4g.com:2095/announce", + "http://bt.okmp3.ru:2710/announce", + "udp://tracker.torrent.eu.org:451/announce", + "http://tracker.mywaifu.best:6969/announce", + "udp://ttk2.nbaonlineservice.com:6969/announce", + "http://tracker.privateseedbox.xyz:2710/announce", + "udp://evan.im:6969/announce", + "https://tracker.yemekyedim.com:443/announce", + "udp://retracker.lanta.me:2710/announce", + "udp://martin-gebhardt.eu:25/announce", + "http://tracker.beeimg.com:6969/announce", + "udp://udp.tracker.projectk.org:23333/announce", + "http://tracker.renfei.net:8080/announce", + "https://tracker.expli.top:443/announce", + "https://tr.nyacat.pw:443/announce", + "udp://tracker.ducks.party:1984/announce", + "udp://extracker.dahrkael.net:6969/announce", + "http://ipv4.rer.lol:2710/announce", + "udp://tracker.plx.im:6969/announce", + "udp://tracker.tvunderground.org.ru:3218/announce", + "http://tracker.tricitytorrents.com:2710/announce", + "udp://open.stealth.si:80/announce", + "udp://tracker.dler.com:6969/announce", + "https://tracker.moeblog.cn:443/announce", + "udp://d40969.acod.regrucolo.ru:6969/announce", + "https://tracker.jdx3.org:443/announce", + "http://ipv6.rer.lol:6969/announce", + "udp://bandito.byterunner.io:6969/announce", + "udp://tracker.gigantino.net:6969/announce", + "http://tracker.netmap.top:6969/announce", + "udp://tracker.yume-hatsuyuki.moe:6969/announce", + "https://tracker.aburaya.live:443/announce", + "udp://tracker.srv00.com:6969/announce", + "udp://open.demonii.com:1337/announce", + "udp://1c.premierzal.ru:6969/announce", + "udp://tracker.fnix.net:6969/announce", + "udp://tracker.kmzs123.cn:17272/announce", + "https://tracker.home.kmzs123.cn:4443/announce", + "udp://tracker-udp.gbitt.info:80/announce", + "udp://tracker.torrust-demo.com:6969/announce", + "udp://tracker.hifimarket.in:2710/announce", + "udp://retracker01-msk-virt.corbina.net:80/announce", + "https://tracker.ghostchu-services.top:443/announce", + "udp://open.dstud.io:6969/announce", + "udp://tracker.therarbg.to:6969/announce", + "udp://tracker.bitcoinindia.space:6969/announce", + "udp://www.torrent.eu.org:451/announce", + "udp://tracker.hifitechindia.com:6969/announce", + "udp://tracker.gmi.gd:6969/announce", + "udp://tracker.skillindia.site:6969/announce", + "http://tracker.ipv6tracker.ru:80/announce", + "udp://tracker.tryhackx.org:6969/announce", + "http://torrent.hificode.in:6969/announce", + "http://open.trackerlist.xyz:80/announce", + "http://taciturn-shadow.spb.ru:6969/announce", + "http://0123456789nonexistent.com:80/announce", + "http://shubt.net:2710/announce", + "udp://tracker.valete.tf:9999/announce", + "https://tracker.zhuqiy.top:443/announce", + "https://tracker.leechshield.link:443/announce", + "http://tracker.tritan.gg:8080/announce", + "udp://t.overflow.biz:6969/announce", + "udp://open.tracker.cl:1337/announce", + "udp://explodie.org:6969/announce", + "udp://exodus.desync.com:6969/announce", + "udp://bt.ktrackers.com:6666/announce", + "udp://wepzone.net:6969/announce", + "udp://tracker2.dler.org:80/announce", + "udp://tracker.theoks.net:6969/announce", + "udp://tracker.ololosh.space:6969/announce", + "udp://tracker.filemail.com:6969/announce", + "udp://tracker.dump.cl:6969/announce", + "udp://tracker.dler.org:6969/announce", + "udp://tracker.bittor.pw:1337/announce", +} + func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metrics, infoHash string, trackers []string) (int, int, error) { leech, seed, err := getPeersFromCache(ctx, r, infoHash) if err != nil { @@ -59,7 +140,12 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metric var peerChan = make(chan peers) var errChan = make(chan error) - for _, tracker := range trackers { + allTrackers := make([]string, 0, len(trackers)+len(additionalTrackers)) + allTrackers = append(allTrackers, trackers...) + allTrackers = append(allTrackers, additionalTrackers...) + allTrackers = utils.StableUniq(allTrackers) + + for _, tracker := range allTrackers { go func(tracker string) { // get peers and seeds from redis first scraper, err := New(tracker) @@ -85,7 +171,7 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metric } var peer peers - for i := 0; i < len(trackers); i++ { + for i := 0; i < len(allTrackers); i++ { select { case <-errChan: // discard error diff --git a/utils/util.go b/utils/util.go index 7f645dc..23dfadc 100644 --- a/utils/util.go +++ b/utils/util.go @@ -19,9 +19,9 @@ func Filter[A any](arr []A, f func(A) bool) []A { return res } -// ParallelMap applies a function to each item in the iterable concurrently +// ParallelFlatMap applies a function to each item in the iterable concurrently // and returns a slice of results. It can handle errors by passing an error handler function. -func ParallelMap[T any, R any](iterable []T, mapper func(item T) ([]R, error), errHandler ...func(error)) []R { +func ParallelFlatMap[T any, R any](iterable []T, mapper func(item T) ([]R, error), errHandler ...func(error)) []R { var itChan = make(chan []R) var errChan = make(chan error) mappedItems := []R{} @@ -89,3 +89,19 @@ func IsValidHTML(input string) bool { _, err := html.Parse(r) return err == nil } + +// FormatBytes formats a byte size into a human-readable string. +// It converts bytes to KB, MB, or GB as appropriate. +func FormatBytes(bytes int64) string { + if bytes < 1024 { + return fmt.Sprintf("%d B", bytes) + } else if bytes < 1024*1024 { + return fmt.Sprintf("%.2f KB", float64(bytes)/1024) + } else if bytes < 1024*1024*1024 { + return fmt.Sprintf("%.2f MB", float64(bytes)/(1024*1024)) + } else if bytes < 1024*1024*1024*1024 { + return fmt.Sprintf("%.2f GB", float64(bytes)/(1024*1024*1024)) + } else { + return fmt.Sprintf("%.2f TB", float64(bytes)/(1024*1024*1024*1024)) + } +}