darklyn/torrent-indexer
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

new: feat: add magnet-metadata-api post processor (#39)

Browse Source 
* new: feat: add magnet-metadata-api post processor

* chg: fix: lint issue

* chg: chore: comment optional containers

* chg: fix: remove redundant check
This commit is contained in:
Felipe Marinho
2025-07-29 12:34:37 -03:00
committed by GitHub
parent e5dea934f1
commit d9141c8df7
14 changed files with 389 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
api/bludv.go
View File

@@ -80,7 +80,7 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrentsBluDV(ctx, i, link)
})
@@ -137,7 +137,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In
// if decoded magnet link is indeed a magnet link, append it
if strings.HasPrefix(magnetLinkDecoded, "magnet:") {
magnetLinks = append(magnetLinks, magnetLinkDecoded)
} else {
} else if !strings.Contains(magnetLinkDecoded, "watch.brplayer") {
fmt.Printf("WARN: link \"%s\" decoding resulted in non-magnet link: %s\n", href, magnetLinkDecoded)
}
})
@@ -211,6 +211,11 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.In
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,

7
api/comando_torrents.go
View File

@@ -94,7 +94,7 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrents(ctx, i, link)
})
@@ -208,6 +208,11 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.Indexed
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,

7
api/comandohds.go
View File

@@ -82,7 +82,7 @@ func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Reques
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrentsComandoHDs(ctx, i, link)
})
@@ -192,6 +192,11 @@ func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]sche
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,

40
api/index.go
View File

@@ -6,6 +6,7 @@ import (
"time"
"github.com/felipemarinho97/torrent-indexer/cache"
"github.com/felipemarinho97/torrent-indexer/magnet"
"github.com/felipemarinho97/torrent-indexer/monitoring"
"github.com/felipemarinho97/torrent-indexer/requester"
"github.com/felipemarinho97/torrent-indexer/schema"
@@ -13,11 +14,12 @@ import (
)
type Indexer struct {
redis *cache.Redis
metrics *monitoring.Metrics
requester *requester.Requster
search *meilisearch.SearchIndexer
postProcessors []PostProcessorFunc
redis *cache.Redis
metrics *monitoring.Metrics
requester *requester.Requster
search *meilisearch.SearchIndexer
magnetMetadataAPI *magnet.MetadataClient
postProcessors []PostProcessorFunc
}
type IndexerMeta struct {
@@ -35,19 +37,27 @@ type Response struct {
type PostProcessorFunc func(*Indexer, *http.Request, []schema.IndexedTorrent) []schema.IndexedTorrent
var GlobalPostProcessors = []PostProcessorFunc{
AddSimilarityCheck, // Jaccard similarity
CleanupTitleWebsites, // Remove website names from titles
AppendAudioTags, // Add (brazilian, eng, etc.) audio tags to titles
SendToSearchIndexer, // Send indexed torrents to Meilisearch
AddSimilarityCheck, // Jaccard similarity
FullfilMissingMetadata, // Fill missing size or title metadata
CleanupTitleWebsites, // Remove website names from titles
AppendAudioTags, // Add (brazilian, eng, etc.) audio tags to titles
SendToSearchIndexer, // Send indexed torrents to Meilisearch
}
func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster, si *meilisearch.SearchIndexer) *Indexer {
func NewIndexers(
redis *cache.Redis,
metrics *monitoring.Metrics,
req *requester.Requster,
si *meilisearch.SearchIndexer,
mc *magnet.MetadataClient,
) *Indexer {
return &Indexer{
redis: redis,
metrics: metrics,
requester: req,
search: si,
postProcessors: GlobalPostProcessors,
redis: redis,
metrics: metrics,
requester: req,
search: si,
magnetMetadataAPI: mc,
postProcessors: GlobalPostProcessors,
}
}

39
api/post_processors.go
View File

@@ -35,6 +35,45 @@ func SendToSearchIndexer(i *Indexer, _ *http.Request, torrents []schema.IndexedT
return torrents
}
// FullfilMissingMetadata fills in missing metadata for indexed torrents
func FullfilMissingMetadata(i *Indexer, r *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
if !i.magnetMetadataAPI.IsEnabled() {
return torrents
}
return utils.ParallelFlatMap(torrents, func(it schema.IndexedTorrent) ([]schema.IndexedTorrent, error) {
if it.Size != "" && it.Title != "" && it.OriginalTitle != "" {
return []schema.IndexedTorrent{it}, nil
}
m, err := i.magnetMetadataAPI.FetchMetadata(r.Context(), it.MagnetLink)
if err != nil {
return []schema.IndexedTorrent{it}, nil
}
// convert size in bytes to a human-readable format
it.Size = utils.FormatBytes(m.Size)
// Use name from metadata if available as it is more accurate
if m.Name != "" {
it.Title = m.Name
}
fmt.Printf("hash: %s get -> size: %s\n", m.InfoHash, it.Size)
// If files are present, add them to the indexed torrent
if len(m.Files) > 0 {
it.Files = make([]schema.File, len(m.Files))
for i, file := range m.Files {
it.Files[i] = schema.File{
Path: file.Path,
Size: utils.FormatBytes(file.Size),
}
}
}
return []schema.IndexedTorrent{it}, nil
})
}
func AddSimilarityCheck(i *Indexer, r *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
q := r.URL.Query().Get("q")

7
api/rede_torrent.go
View File

@@ -80,7 +80,7 @@ func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Reque
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrentsRedeTorrent(ctx, i, link)
})
@@ -216,6 +216,11 @@ func getTorrentsRedeTorrent(ctx context.Context, i *Indexer, link string) ([]sch
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,

7
api/starck_filmes.go
View File

@@ -79,7 +79,7 @@ func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Requ
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrentStarckFilmes(ctx, i, link)
})
@@ -192,6 +192,11 @@ func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]sch
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,

9
api/torrent_dos_filmes.go
View File

@@ -21,7 +21,7 @@ var torrent_dos_filmes = IndexerMeta{
Label: "torrent_dos_filmes",
URL: "https://torrentdosfilmes.se/",
SearchURL: "?s=",
PagePattern: "page/%s",
PagePattern: "category/dublado/page/%s",
}
func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.Request) {
@@ -79,7 +79,7 @@ func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.
})
// extract each torrent link
indexedTorrents := utils.ParallelMap(links, func(link string) ([]schema.IndexedTorrent, error) {
indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
return getTorrentsTorrentDosFilmes(ctx, i, link)
})
@@ -186,6 +186,11 @@ func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) (
if len(size) == len(magnetLinks) {
mySize = size[it]
}
if mySize == "" {
go func() {
_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
}()
}
ixt := schema.IndexedTorrent{
Title: releaseTitle,

61
docker-compose.yml
View File

@@ -11,9 +11,16 @@ services:
- indexer
environment:
- REDIS_HOST=redis
- MEILISEARCH_ADDRESS=http://meilisearch:7700
- MEILISEARCH_KEY=my-secret-key
- FLARESOLVERR_ADDRESS=http://flaresolverr:8191
## Meilisearch configuration (optional)
# - MEILISEARCH_ADDRESS=http://meilisearch:7700
# - MEILISEARCH_KEY=my-secret-key
## Magnet Metadata API configuration (optional)
# - MAGNET_METADATA_API_ENABLED=false
# - MAGNET_METADATA_API_ADDRESS=http://magnet-metadata-api:8080
# - MAGNET_METADATA_API_TIMEOUT_SECONDS=10
redis:
image: redis:alpine
@@ -22,17 +29,45 @@ services:
networks:
- indexer
# This container is not necessary for the indexer to work,
# deploy if you want to use the search feature
meilisearch:
image: getmeili/meilisearch:latest
container_name: meilisearch
restart: unless-stopped
networks:
- indexer
environment:
- MEILI_NO_ANALYTICS=true
- MEILI_MASTER_KEY=my-secret-key
##### MEILISEARCH #####
## This container is not necessary for the indexer to work,
## deploy if you want to use the search feature
#
# meilisearch:
# image: getmeili/meilisearch:latest
# container_name: meilisearch
# restart: unless-stopped
# networks:
# - indexer
# environment:
# - MEILI_NO_ANALYTICS=true
# - MEILI_MASTER_KEY=my-secret-key
##### MAGNET METADATA API #####
## This container is not necessary for the indexer to work,
## deploy if you want to fetch metadata from p2p network
## CAUTION: Never deploy this container on a cloud server (AWS, GCP, Azure, Oracle), or you will get banned!
#
# magnet-metadata-api:
# image: felipemarinho97/magnet-metadata-api:latest
# container_name: magnet-metadata-api
# restart: unless-stopped
# ports:
# - "8999:8080"
# - "42069:42069"
# networks:
# - indexer
# environment:
# - PORT=8080
# - REDIS_URL=redis://redis:6379
# - CACHE_DIR=/home/torrent/cache
# - ENABLE_DOWNLOADS=false
# - DOWNLOAD_BASE_URL=http://localhost:8999
# - CLIENT_PORT=42069
# - SEEDING_ENABLED=false
# - FALLBACK_INITIAL_CHUNK_SIZE_KB=24
# volumes:
# - ./magnet-metadata-cache:/home/torrent/cache
networks:
indexer:

113
magnet/magnet_metadata_api.go Normal file
View File

@@ -0,0 +1,113 @@
package magnet
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/felipemarinho97/torrent-indexer/cache"
)
type MetadataRequest struct {
MagnetURI string `json:"magnet_uri"`
}
type TorrentFile struct {
Path string `json:"path"`
Size int64 `json:"size"`
Offset int64 `json:"offset"`
}
type MetadataResponse struct {
InfoHash string `json:"info_hash"`
Name string `json:"name"`
Size int64 `json:"size"`
Files []TorrentFile `json:"files"`
CreatedBy string `json:"created_by"`
CreatedAt time.Time `json:"created_at"`
Comment string `json:"comment"`
Trackers []string `json:"trackers"`
DownloadURL string `json:"download_url"`
}
type MetadataClient struct {
baseURL string
httpClient *http.Client
c *cache.Redis
}
func NewClient(baseURL string, timeout time.Duration, c *cache.Redis) *MetadataClient {
return &MetadataClient{
baseURL: baseURL,
httpClient: &http.Client{
Timeout: timeout,
Transport: &http.Transport{
MaxIdleConns: 100,
IdleConnTimeout: 30 * time.Second,
ForceAttemptHTTP2: true,
},
},
c: c,
}
}
func (c *MetadataClient) IsEnabled() bool {
return c != nil && c.baseURL != ""
}
func (c *MetadataClient) FetchMetadata(ctx context.Context, magnetURI string) (*MetadataResponse, error) {
if !c.IsEnabled() {
return nil, fmt.Errorf("magnet metadata API is not enabled")
}
// Check cache first
m, err := ParseMagnetUri(magnetURI)
if err != nil {
return nil, fmt.Errorf("failed to parse magnet URI: %w", err)
}
cacheKey := fmt.Sprintf("metadata:%s", m.InfoHash)
cachedData, err := c.c.Get(ctx, cacheKey)
if err == nil && cachedData != nil {
var cachedMetadata MetadataResponse
if err := json.Unmarshal(cachedData, &cachedMetadata); err == nil {
return &cachedMetadata, nil
}
}
reqBody := MetadataRequest{MagnetURI: magnetURI}
body, err := json.Marshal(reqBody)
if err != nil {
return nil, fmt.Errorf("failed to marshal request body: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/api/v1/metadata", bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send POST request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("API responded with status: %s", resp.Status)
}
var metadata MetadataResponse
if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
return nil, fmt.Errorf("failed to decode response: %w", err)
}
// Cache the metadata response
cacheData, err := json.Marshal(metadata)
if err == nil {
_ = c.c.SetWithExpiration(ctx, cacheKey, cacheData, 7*24*time.Hour)
}
return &metadata, nil
}

15
main.go
View File

@@ -4,9 +4,12 @@ import (
"fmt"
"net/http"
"os"
"strconv"
"time"
handler "github.com/felipemarinho97/torrent-indexer/api"
"github.com/felipemarinho97/torrent-indexer/cache"
"github.com/felipemarinho97/torrent-indexer/magnet"
"github.com/felipemarinho97/torrent-indexer/monitoring"
"github.com/felipemarinho97/torrent-indexer/public"
"github.com/felipemarinho97/torrent-indexer/requester"
@@ -19,6 +22,16 @@ import (
func main() {
redis := cache.NewRedis()
searchIndex := meilisearch.NewSearchIndexer(os.Getenv("MEILISEARCH_ADDRESS"), os.Getenv("MEILISEARCH_KEY"), "torrents")
var magnetMetadataAPI *magnet.MetadataClient
if os.Getenv("MAGNET_METADATA_API_ENABLED") == "true" {
timeout := 10 * time.Second
if v := os.Getenv("MAGNET_METADATA_API_TIMEOUT_SECONDS"); v != "" {
if t, err := strconv.Atoi(v); err == nil {
timeout = time.Duration(t) * time.Second
}
}
magnetMetadataAPI = magnet.NewClient(os.Getenv("MAGNET_METADATA_API_ADDRESS"), timeout, redis)
}
metrics := monitoring.NewMetrics()
metrics.Register()
@@ -39,7 +52,7 @@ func main() {
fmt.Println(err)
}
indexers := handler.NewIndexers(redis, metrics, req, searchIndex)
indexers := handler.NewIndexers(redis, metrics, req, searchIndex, magnetMetadataAPI)
search := handler.NewMeilisearchHandler(searchIndex)
indexerMux := http.NewServeMux()

6
schema/indexed_torrent.go
View File

@@ -14,7 +14,13 @@ type IndexedTorrent struct {
InfoHash string `json:"info_hash"`
Trackers []string `json:"trackers"`
Size string `json:"size"`
Files []File `json:"files,omitempty"`
LeechCount int `json:"leech_count"`
SeedCount int `json:"seed_count"`
Similarity float32 `json:"similarity"`
}
type File struct {
Path string `json:"path"`
Size string `json:"size"`
}

90
scrape/info.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/felipemarinho97/torrent-indexer/cache"
"github.com/felipemarinho97/torrent-indexer/monitoring"
"github.com/felipemarinho97/torrent-indexer/utils"
)
type peers struct {
@@ -45,6 +46,86 @@ func setPeersToCache(ctx context.Context, r *cache.Redis, infoHash string, peer,
return nil
}
var additionalTrackers = []string{
"udp://tracker.opentrackr.org:1337/announce",
"udp://p4p.arenabg.com:1337/announce",
"udp://retracker.hotplug.ru:2710/announce",
"http://tracker.bt4g.com:2095/announce",
"http://bt.okmp3.ru:2710/announce",
"udp://tracker.torrent.eu.org:451/announce",
"http://tracker.mywaifu.best:6969/announce",
"udp://ttk2.nbaonlineservice.com:6969/announce",
"http://tracker.privateseedbox.xyz:2710/announce",
"udp://evan.im:6969/announce",
"https://tracker.yemekyedim.com:443/announce",
"udp://retracker.lanta.me:2710/announce",
"udp://martin-gebhardt.eu:25/announce",
"http://tracker.beeimg.com:6969/announce",
"udp://udp.tracker.projectk.org:23333/announce",
"http://tracker.renfei.net:8080/announce",
"https://tracker.expli.top:443/announce",
"https://tr.nyacat.pw:443/announce",
"udp://tracker.ducks.party:1984/announce",
"udp://extracker.dahrkael.net:6969/announce",
"http://ipv4.rer.lol:2710/announce",
"udp://tracker.plx.im:6969/announce",
"udp://tracker.tvunderground.org.ru:3218/announce",
"http://tracker.tricitytorrents.com:2710/announce",
"udp://open.stealth.si:80/announce",
"udp://tracker.dler.com:6969/announce",
"https://tracker.moeblog.cn:443/announce",
"udp://d40969.acod.regrucolo.ru:6969/announce",
"https://tracker.jdx3.org:443/announce",
"http://ipv6.rer.lol:6969/announce",
"udp://bandito.byterunner.io:6969/announce",
"udp://tracker.gigantino.net:6969/announce",
"http://tracker.netmap.top:6969/announce",
"udp://tracker.yume-hatsuyuki.moe:6969/announce",
"https://tracker.aburaya.live:443/announce",
"udp://tracker.srv00.com:6969/announce",
"udp://open.demonii.com:1337/announce",
"udp://1c.premierzal.ru:6969/announce",
"udp://tracker.fnix.net:6969/announce",
"udp://tracker.kmzs123.cn:17272/announce",
"https://tracker.home.kmzs123.cn:4443/announce",
"udp://tracker-udp.gbitt.info:80/announce",
"udp://tracker.torrust-demo.com:6969/announce",
"udp://tracker.hifimarket.in:2710/announce",
"udp://retracker01-msk-virt.corbina.net:80/announce",
"https://tracker.ghostchu-services.top:443/announce",
"udp://open.dstud.io:6969/announce",
"udp://tracker.therarbg.to:6969/announce",
"udp://tracker.bitcoinindia.space:6969/announce",
"udp://www.torrent.eu.org:451/announce",
"udp://tracker.hifitechindia.com:6969/announce",
"udp://tracker.gmi.gd:6969/announce",
"udp://tracker.skillindia.site:6969/announce",
"http://tracker.ipv6tracker.ru:80/announce",
"udp://tracker.tryhackx.org:6969/announce",
"http://torrent.hificode.in:6969/announce",
"http://open.trackerlist.xyz:80/announce",
"http://taciturn-shadow.spb.ru:6969/announce",
"http://0123456789nonexistent.com:80/announce",
"http://shubt.net:2710/announce",
"udp://tracker.valete.tf:9999/announce",
"https://tracker.zhuqiy.top:443/announce",
"https://tracker.leechshield.link:443/announce",
"http://tracker.tritan.gg:8080/announce",
"udp://t.overflow.biz:6969/announce",
"udp://open.tracker.cl:1337/announce",
"udp://explodie.org:6969/announce",
"udp://exodus.desync.com:6969/announce",
"udp://bt.ktrackers.com:6666/announce",
"udp://wepzone.net:6969/announce",
"udp://tracker2.dler.org:80/announce",
"udp://tracker.theoks.net:6969/announce",
"udp://tracker.ololosh.space:6969/announce",
"udp://tracker.filemail.com:6969/announce",
"udp://tracker.dump.cl:6969/announce",
"udp://tracker.dler.org:6969/announce",
"udp://tracker.bittor.pw:1337/announce",
}
func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metrics, infoHash string, trackers []string) (int, int, error) {
leech, seed, err := getPeersFromCache(ctx, r, infoHash)
if err != nil {
@@ -59,7 +140,12 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metric
var peerChan = make(chan peers)
var errChan = make(chan error)
for _, tracker := range trackers {
allTrackers := make([]string, 0, len(trackers)+len(additionalTrackers))
allTrackers = append(allTrackers, trackers...)
allTrackers = append(allTrackers, additionalTrackers...)
allTrackers = utils.StableUniq(allTrackers)
for _, tracker := range allTrackers {
go func(tracker string) {
// get peers and seeds from redis first
scraper, err := New(tracker)
@@ -85,7 +171,7 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metric
}
var peer peers
for i := 0; i < len(trackers); i++ {
for i := 0; i < len(allTrackers); i++ {
select {
case <-errChan:
// discard error

20
utils/util.go
View File

@@ -19,9 +19,9 @@ func Filter[A any](arr []A, f func(A) bool) []A {
return res
}
// ParallelMap applies a function to each item in the iterable concurrently
// ParallelFlatMap applies a function to each item in the iterable concurrently
// and returns a slice of results. It can handle errors by passing an error handler function.
func ParallelMap[T any, R any](iterable []T, mapper func(item T) ([]R, error), errHandler ...func(error)) []R {
func ParallelFlatMap[T any, R any](iterable []T, mapper func(item T) ([]R, error), errHandler ...func(error)) []R {
var itChan = make(chan []R)
var errChan = make(chan error)
mappedItems := []R{}
@@ -89,3 +89,19 @@ func IsValidHTML(input string) bool {
_, err := html.Parse(r)
return err == nil
}
// FormatBytes formats a byte size into a human-readable string.
// It converts bytes to KB, MB, or GB as appropriate.
func FormatBytes(bytes int64) string {
if bytes < 1024 {
return fmt.Sprintf("%d B", bytes)
} else if bytes < 1024*1024 {
return fmt.Sprintf("%.2f KB", float64(bytes)/1024)
} else if bytes < 1024*1024*1024 {
return fmt.Sprintf("%.2f MB", float64(bytes)/(1024*1024))
} else if bytes < 1024*1024*1024*1024 {
return fmt.Sprintf("%.2f GB", float64(bytes)/(1024*1024*1024))
} else {
return fmt.Sprintf("%.2f TB", float64(bytes)/(1024*1024*1024*1024))
}
}