Chore/update golang to 1.24 + add riscv64 support (#40)

* chg: chore: bump golang and deps

* chg: chore: add riscv64 support

* chg: chore: update goreleaser

* chg: chore: disable cgo

* chg: chore: tidy up deps

* chg: chore: add build tags

.github/workflows/pipeline.dockerfile

@@ -0,0 +1,27 @@
+#####################################################
+### Copy platform specific binary
+FROM bash AS copy-binary
+ARG TARGETPLATFORM
+
+RUN echo "Target Platform = ${TARGETPLATFORM}"
+
+COPY dist .
+RUN if [ "$TARGETPLATFORM" = "linux/amd64" ];  then cp torrentindexer_linux_amd64_linux_amd64_v1/torrent-indexer /torrent-indexer; fi
+RUN if [ "$TARGETPLATFORM" = "linux/386" ];  then cp torrentindexer_linux_386_linux_386_sse2/torrent-indexer /torrent-indexer; fi
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ];  then cp torrentindexer_linux_arm64_linux_arm64_v8.0/torrent-indexer /torrent-indexer; fi
+RUN if [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then cp torrentindexer_linux_arm_linux_arm_6/torrent-indexer /torrent-indexer; fi
+RUN if [ "$TARGETPLATFORM" = "linux/arm/v7" ]; then cp torrentindexer_linux_arm_linux_arm_7/torrent-indexer /torrent-indexer; fi
+RUN if [ "$TARGETPLATFORM" = "linux/riscv64" ]; then cp torrentindexer_linux_riscv64_linux_riscv64_rva20u64/torrent-indexer /torrent-indexer; fi
+RUN chmod +x /torrent-indexer
+
+
+#####################################################
+### Build Final Image
+FROM alpine AS release
+LABEL maintainer="felipevm97@gmail.com"
+
+COPY --from=copy-binary /torrent-indexer /app/
+
+WORKDIR /app
+
+ENTRYPOINT ["/app/torrent-indexer"]

.github/workflows/pipeline.yml

@@ -0,0 +1,158 @@
+name: "Pipeline: Test, Lint, Build"
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "v*"
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  go-lint:
+    name: Lint Go code
+    runs-on: ubuntu-latest
+    container: golang:1.24
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Config workspace folder as trusted
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE;  git describe --dirty --always --tags
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v4
+        with:
+          version: latest
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          args: --timeout 2m
+
+      - run: go mod tidy
+      - name: Verify no changes from go mod tidy
+        run: |
+          git status --porcelain
+          if [ -n "$(git status --porcelain)" ]; then
+            echo 'To fix this check, run "make format" and commit the changes'
+            exit 1
+          fi
+
+  go:
+    name: Test Go code
+    runs-on: ubuntu-latest
+    container: golang:1.24
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Config workspace folder as trusted
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE;  git describe --dirty --always --tags
+
+      - name: Download dependencies
+        if: steps.cache-go.outputs.cache-hit != 'true'
+        continue-on-error: ${{contains(matrix.go_version, 'beta') || contains(matrix.go_version, 'rc')}}
+        run: go mod download
+
+      - name: Test
+        continue-on-error: ${{contains(matrix.go_version, 'beta') || contains(matrix.go_version, 'rc')}}
+        run: go test -shuffle=on -race -cover ./... -v
+
+  binaries:
+    name: Build binaries
+    needs: [go, go-lint]
+    runs-on: ubuntu-latest
+    container: goreleaser/goreleaser:v2.11.1
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Config workspace folder as trusted
+        run: git config --global --add safe.directory $GITHUB_WORKSPACE;  git describe --dirty --always --tags
+
+      - name: Run GoReleaser - SNAPSHOT
+        if: startsWith(github.ref, 'refs/tags/') != true
+        run: goreleaser release --clean --skip=publish --snapshot
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run GoReleaser - RELEASE
+        if: startsWith(github.ref, 'refs/tags/')
+        run: goreleaser release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: binaries
+          path: |
+            dist
+            !dist/*.tar.gz
+            !dist/*.zip
+          retention-days: 7
+
+  docker:
+    name: Build and publish Docker images
+    needs: [binaries]
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_IMAGE: ${{secrets.DOCKER_IMAGE}}
+    steps:
+      - name: Set up QEMU
+        id: qemu
+        uses: docker/setup-qemu-action@v3
+        if: env.DOCKER_IMAGE != ''
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        if: env.DOCKER_IMAGE != ''
+
+      - uses: actions/checkout@v4
+        if: env.DOCKER_IMAGE != ''
+
+      - uses: actions/download-artifact@v4
+        if: env.DOCKER_IMAGE != ''
+        with:
+          name: binaries
+          path: dist
+
+      - name: Login to Docker Hub
+        if: env.DOCKER_IMAGE != ''
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        if: env.DOCKER_IMAGE != ''
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
+        if: env.DOCKER_IMAGE != ''
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          labels: |
+            maintainer=felipemarinho97
+          images: |
+            name=${{secrets.DOCKER_IMAGE}}
+            name=ghcr.io/${{ github.repository }}
+          tags: |
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=raw,value=develop,enable={{is_default_branch}}
+
+      - name: Build and Push
+        if: env.DOCKER_IMAGE != ''
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: .github/workflows/pipeline.dockerfile
+          platforms: linux/amd64,linux/386,linux/arm/v6,linux/arm/v7,linux/arm64,linux/riscv64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

.golangci.yml

@@ -0,0 +1,4 @@
+issues:
+  exclude-files:
+    - scrape.go
+    - infohash.go

.goreleaser.yml

@@ -0,0 +1,150 @@
+# GoReleaser config
+project_name: torrent-indexer
+version: 2
+
+builds:
+  - id: torrentindexer_linux_amd64
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    goarch:
+      - amd64
+    flags:
+      - -tags=netgo
+    ldflags:
+      - "-extldflags '-static -lz'"
+      - -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
+
+  - id: torrentindexer_linux_386
+    env:
+      - CGO_ENABLED=0
+      - PKG_CONFIG_PATH=/i386/lib/pkgconfig
+    goos:
+      - linux
+    goarch:
+      - "386"
+    flags:
+      - -tags=netgo
+    ldflags:
+      - "-extldflags '-static'"
+      - -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
+
+  - id: torrentindexer_linux_arm
+    env:
+      - CGO_ENABLED=0
+      - CC=arm-linux-gnueabi-gcc
+      - CXX=arm-linux-gnueabi-g++
+      - PKG_CONFIG_PATH=/arm/lib/pkgconfig
+    goos:
+      - linux
+    goarch:
+      - arm
+    goarm:
+      - "5"
+      - "6"
+      - "7"
+    flags:
+      - -tags=netgo
+    ldflags:
+      - "-extldflags '-static'"
+      - -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
+
+  - id: torrentindexer_linux_arm64
+    env:
+      - CGO_ENABLED=0
+      - CC=aarch64-linux-gnu-gcc
+      - CXX=aarch64-linux-gnu-g++
+      - PKG_CONFIG_PATH=/arm64/lib/pkgconfig
+    goos:
+      - linux
+    goarch:
+      - arm64
+    flags:
+      - -tags=netgo
+    ldflags:
+      - "-extldflags '-static'"
+      - -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
+  
+  - id: torrentindexer_linux_riscv64
+    env:
+      - CGO_ENABLED=0
+      - CC=riscv64-linux-gnu-gcc
+      - CXX=riscv64-linux-gnu-g++
+      - PKG_CONFIG_PATH=/riscv64/lib/pkgconfig
+    goos:
+      - linux
+    goarch:
+      - riscv64
+    flags:
+      - -tags=netgo
+    ldflags:
+      - "-extldflags '-static'"
+      - -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
+
+  - id: torrentindexer_windows_386
+    env:
+      - CGO_ENABLED=0
+      - CC=i686-w64-mingw32-gcc
+      - CXX=i686-w64-mingw32-g++
+      - PKG_CONFIG_PATH=/mingw32/lib/pkgconfig
+    goos:
+      - windows
+    goarch:
+      - "386"
+    flags:
+      - -tags=netgo
+    ldflags:
+      - "-extldflags '-static'"
+      - -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
+
+  - id: torrentindexer_windows_amd64
+    env:
+      - CGO_ENABLED=0
+      - CC=x86_64-w64-mingw32-gcc
+      - CXX=x86_64-w64-mingw32-g++
+      - PKG_CONFIG_PATH=/mingw64/lib/pkgconfig
+    goos:
+      - windows
+    goarch:
+      - amd64
+    flags:
+      - -tags=netgo
+    ldflags:
+      - "-extldflags '-static'"
+      - -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
+
+  - id: torrentindexer_darwin_amd64
+    env:
+      - CGO_ENABLED=0
+      - CC=o64-clang
+      - CXX=o64-clang++
+      - PKG_CONFIG_PATH=/darwin/lib/pkgconfig
+    goos:
+      - darwin
+    goarch:
+      - amd64
+    flags:
+      - -tags=netgo
+    ldflags:
+      - -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
+
+archives:
+  - format_overrides:
+      - goos: windows
+        formats: zip
+
+checksum:
+  name_template: "{{ .ProjectName }}_checksums.txt"
+
+snapshot:
+  version_template: "{{ .Tag }}-SNAPSHOT"
+
+release:
+  draft: true
+
+changelog:
+  # sort: asc
+  filters:
+    exclude:
+      - "^docs:"

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19 as builder
+FROM golang:1.24 AS builder
 
 WORKDIR /go/src/app
 COPY . .
@@ -8,6 +8,7 @@ RUN go install -v ./...
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app .
 
 FROM alpine:latest
+LABEL maintainer="felipevm97@gmail.com"
 
 RUN apk --no-cache add ca-certificates
 

LICENSE

@@ -0,0 +1,621 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS

Makefile

@@ -0,0 +1,9 @@
+
+build:
+	docker build -t torrent-indexer .
+
+lint:
+	go run github.com/golangci/golangci-lint/cmd/golangci-lint@latest run -v --timeout 5m
+
+run:
+	go run main.go

README.md

@@ -1,12 +1,213 @@
 # torrent-indexer
 
+[![](https://dcbadge.limes.pink/api/server/7wqNywmpQW)](https://discord.gg/7wqNywmpQW)
+
 This is a simple torrent indexer that can be used to index torrents from HTML pages. It is written in Golang and uses Redis as a cache.
 
 ## Test it
 
-Visit [https://vlambdas.oci.darklyn.online/](https://vlambdas.oci.darklyn.online/) to test it.
+Visit [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org/) to test it.
 
 ## Supported sites
 
 - [comando-torrents](https://comando.la/)
-- [bludv](https://bludvfilmes.tv/)
+- [bludv](https://bludvfilmes.tv/)
+- [torrent-dos-filmes](https://torrentdosfilmes.se/)
+- [starck-filmes](https://www.starckfilmes.online/)
+- [comandohds](https://comandohds.org/)
+- [rede-torrent](https://redetorrent.com/)
+
+## Deploy
+
+If you have Docker + docker-compose installed, you can deploy it using the following command:
+
+```bash
+curl -s https://raw.githubusercontent.com/felipemarinho97/torrent-indexer/main/docker-compose.yml > docker-compose.yml
+docker-compose up -d
+```
+
+The server will be available at [http://localhost:8080/](http://localhost:8080/).
+
+## Configuration
+
+You can configure the server using the following environment variables:
+  
+- `PORT`: (optional) The port that the server will listen to. Default: `7006`
+- `FLARESOLVERR_ADDRESS`: (optional) The address of the FlareSolverr instance. Default: `N/A`
+- `MEILISEARCH_ADDRESS`: (optional) The address of the MeiliSearch instance. Default: `N/A`
+- `MEILISEARCH_KEY`: (optional) The API key of the MeiliSearch instance. Default: `N/A`
+- `REDIS_HOST`: (optional) The address of the Redis instance. Default: `localhost`
+- `SHORT_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the short-lived cache in duration format. Default: `30m`
+    - This cache is used to cache homepage or search results.
+    - Example: `30m`, `1h`, `1h30m`, `1h30m30s`
+- `LONG_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the long-lived cache in duration format. Default: `7d`
+    - This cache is used to store the torrent webpages (posts). You can set it to a higher value because the torrent pages are not updated frequently.
+
+## Integrating with Jackett
+
+You can integrate this indexer with Jackett by adding a new Torznab custom indexer. Here is an example of how to do it for the `bludv` indexer:
+
+```yaml
+---
+id: bludv_indexer
+name: BluDV Indexer
+description: "BluDV - Custom indexer on from torrent-indexer"
+language: pt-BR
+type: public
+encoding: UTF-8
+links:
+  - http://localhost:8080/
+
+caps:
+  categorymappings:
+    - { id: Movie, cat: Movies, desc: "Movies" }
+    - { id: TV, cat: TV, desc: "TV" }
+
+  modes:
+    search: [q]
+    tv-search: [q, season, ep]
+    movie-search: [q]
+  allowrawsearch: true
+
+settings: []
+
+search:
+  paths:
+    - path: "indexers/bludv?filter_results=true&q={{ .Keywords }}"
+      response:
+        type: json
+
+  keywordsfilters:
+    - name: tolower
+
+  rows:
+    selector: $.results
+    count:
+      selector: $.count
+
+  fields:
+    _id:
+      selector: title
+    download:
+      selector: magnet_link
+    title:
+      selector: title
+    description:
+      selector: original_title
+    details:
+      selector: details
+    infohash:
+      selector: info_hash
+    date:
+      selector: date
+    size:
+      selector: size
+    seeders:
+      selector: seed_count
+    leechers:
+      selector: leech_count
+    imdb:
+      selector: imdb
+    category_is_tv_show:
+      selector: title
+      filters:
+        - name: regexp
+          args: "\\b(S\\d+(?:E\\d+)?)\\b"
+    category:
+      text: "{{ if .Result.category_is_tv_show }}TV{{ else }}Movie{{ end }}"
+# json engine n/a
+```
+
+## Integrating with Prowlarr
+
+You can integrate this indexer with Prowlarr by adding a custom definition. See [Adding a custom YML definition](https://wiki.servarr.com/prowlarr/indexers#adding-a-custom-yml-definition).
+
+```yaml
+---
+id: torrent-indexer
+name: Torrent Indexer
+description: "Indexing Brazilian Torrent websites into structured data. github.com/felipemarinho97/torrent-indexer"
+language: pt-BR
+type: public
+encoding: UTF-8
+links:
+  - http://localhost:8080/
+
+caps:
+  categories:
+    Movies: Movies
+    TV: TV
+
+  modes:
+    search: [q]
+    tv-search: [q, season]
+    movie-search: [q]
+
+settings:
+  - name: indexer
+    type: select
+    label: Indexer
+    default: bludv
+    options:
+      bludv: BLUDV
+      comando_torrents: Comando Torrents
+      torrent-dos-filmes: Torrent dos Filmes
+      comandohds: Comando HDs
+      starck-filmes: Starck Filmes
+      rede_torrent: Rede Torrent
+
+search:
+  paths:
+    - path: "/indexers/{{ .Config.indexer }}"
+      response:
+        type: json
+  inputs:
+    filter_results: "true"
+    q: "{{ .Keywords }}"
+  keywordsfilters:
+    - name: tolower
+    - name: re_replace
+      args: ["(?i)(S0)(\\d{1,2})$", "temporada $2"]
+    - name: re_replace
+      args: ["(?i)(S)(\\d{1,3})$", "temporada $2"]
+
+  rows:
+    selector: $.results
+    count:
+      selector: $.count
+
+  fields:
+    download:
+      selector: magnet_link
+    title:
+      selector: title
+    description:
+      selector: original_title
+    details:
+      selector: details
+    infohash:
+      selector: info_hash
+    date:
+      selector: date
+    size:
+      selector: size
+    seeders:
+      selector: seed_count
+    leechers:
+      selector: leech_count
+    imdb:
+      selector: imdb
+    category_is_tv_show:
+      selector: title
+      filters:
+        - name: regexp
+          args: "\\b(S\\d+(?:E\\d+)?)\\b"
+    category:
+      text: "{{ if .Result.category_is_tv_show }}TV{{ else }}Movies{{ end }}"
+```
+
+# Warning
+
+The instance running at [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org/) is my personal instance and it is not guaranteed to be up all the time. Also, for better availability, I recommend deploying your own instance because the Cloudflare protection may block requests from indexed sites if too many requests are made in a short period of time from the same IP.
+
+If I notice that the instance is being used a lot, I may block requests from Jackett to avoid overloading the server without prior notice.

api/bludv.go

@@ -6,46 +6,69 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
-	"regexp"
 	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
+
 	"github.com/felipemarinho97/torrent-indexer/magnet"
 	"github.com/felipemarinho97/torrent-indexer/schema"
 	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
 )
 
 var bludv = IndexerMeta{
-	URL:       "https://bludvfilmes.tv/",
-	SearchURL: "?s=",
+	Label:       "bludv",
+	URL:         "https://bludv.xyz/",
+	SearchURL:   "?s=",
+	PagePattern: "page/%s",
 }
 
 func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	metadata := bludv
+
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
+	}()
+
 	ctx := r.Context()
-	// supported query params: q, season, episode
+	// supported query params: q, season, episode, page, filter_results
 	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
 
 	// URL encode query param
 	q = url.QueryEscape(q)
-	url := bludv.URL
-	if q != "" {
-		url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q)
+	url := metadata.URL
+	if page != "" {
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
+	} else {
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
 	}
 
 	fmt.Println("URL:>", url)
-	resp, err := http.Get(url)
+	resp, err := i.requester.GetDocument(ctx, url)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
-		json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
-	defer resp.Body.Close()
+	defer resp.Close()
 
-	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	doc, err := goquery.NewDocumentFromReader(resp)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
-		json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 
@@ -56,38 +79,29 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []IndexedTorrent)
-	var errChan = make(chan error)
-	indexedTorrents := []IndexedTorrent{}
-	for _, link := range links {
-		go func(link string) {
-			torrents, err := getTorrentsBluDV(ctx, i, link)
-			if err != nil {
-				fmt.Println(err)
-				errChan <- err
-			}
-			itChan <- torrents
-		}(link)
-	}
+	// extract each torrent link
+	indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrentsBluDV(ctx, i, link)
+	})
 
-	for i := 0; i < len(links); i++ {
-		select {
-		case torrents := <-itChan:
-			indexedTorrents = append(indexedTorrents, torrents...)
-		case err := <-errChan:
-			fmt.Println(err)
-		}
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
 	}
 
 	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
+	err = json.NewEncoder(w).Encode(Response{
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
 	})
+	if err != nil {
+		fmt.Println(err)
+	}
 }
 
-func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
-	var indexedTorrents []IndexedTorrent
+func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+	var indexedTorrents []schema.IndexedTorrent
 	doc, err := getDocument(ctx, i, link)
 	if err != nil {
 		return nil, err
@@ -96,22 +110,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
 	article := doc.Find(".post")
 	title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1)
 	textContent := article.Find("div.content")
-	// div itemprop="datePublished"
-	datePublished := strings.TrimSpace(article.Find("div[itemprop=\"datePublished\"]").Text())
-	// pattern: 10 de setembro de 2021
-	re := regexp.MustCompile(`(\d{2}) de (\w+) de (\d{4})`)
-	matches := re.FindStringSubmatch(datePublished)
-	var date time.Time
-	if len(matches) > 0 {
-		day := matches[1]
-		month := matches[2]
-		year := matches[3]
-		datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
-		date, err = time.Parse("2006-01-02", datePublished)
-		if err != nil {
-			return nil, err
-		}
-	}
+	date := getPublishedDate(doc)
 	magnets := textContent.Find("a[href^=\"magnet\"]")
 	var magnetLinks []string
 	magnets.Each(func(i int, s *goquery.Selection) {
@@ -119,6 +118,30 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
 		magnetLinks = append(magnetLinks, magnetLink)
 	})
 
+	adwareLinks := textContent.Find("a[href^=\"https://www.seuvideo.xyz\"]")
+	adwareLinks.Each(func(_ int, s *goquery.Selection) {
+		href, _ := s.Attr("href")
+		// extract querysting "id" from url
+		parsedUrl, err := url.Parse(href)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		magnetLink := parsedUrl.Query().Get("id")
+		magnetLinkDecoded, err := utils.DecodeAdLink(magnetLink)
+		if err != nil {
+			fmt.Printf("failed to decode ad link \"%s\": %v\n", href, err)
+			return
+		}
+
+		// if decoded magnet link is indeed a magnet link, append it
+		if strings.HasPrefix(magnetLinkDecoded, "magnet:") {
+			magnetLinks = append(magnetLinks, magnetLinkDecoded)
+		} else if !strings.Contains(magnetLinkDecoded, "watch.brplayer") {
+			fmt.Printf("WARN: link \"%s\" decoding resulted in non-magnet link: %s\n", href, magnetLinkDecoded)
+		}
+	})
+
 	var audio []schema.Audio
 	var year string
 	var size []string
@@ -142,7 +165,10 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
 		text := s.Text()
 
 		audio = append(audio, findAudioFromText(text)...)
-		year = findYearFromText(text, title)
+		y := findYearFromText(text, title)
+		if y != "" {
+			year = y
+		}
 		size = append(size, findSizesFromText(text)...)
 	})
 
@@ -150,16 +176,15 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
 	imdbLink := ""
 	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
 		link, _ := s.Attr("href")
-		re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
-		matches := re.FindStringSubmatch(link)
-		if len(matches) > 0 {
-			imdbLink = matches[0]
+		_imdbLink, err := getIMDBLink(link)
+		if err == nil {
+			imdbLink = _imdbLink
 		}
 	})
 
-	size = stableUniq(size)
+	size = utils.StableUniq(size)
 
-	var chanIndexedTorrent = make(chan IndexedTorrent)
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
 
 	// for each magnet link, create a new indexed torrent
 	for it, magnetLink := range magnetLinks {
@@ -172,21 +197,9 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
 			releaseTitle := magnet.DisplayName
 			infoHash := magnet.InfoHash.String()
 			trackers := magnet.Trackers
-			magnetAudio := []schema.Audio{}
-			if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
-				magnetAudio = append(magnetAudio, audio...)
-			} else if len(audio) > 1 {
-				// remove portuguese audio, and append to magnetAudio
-				for _, a := range audio {
-					if a != schema.AudioPortuguese {
-						magnetAudio = append(magnetAudio, a)
-					}
-				}
-			} else {
-				magnetAudio = append(magnetAudio, audio...)
-			}
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
 
-			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, infoHash, trackers)
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
 			if err != nil {
 				fmt.Println(err)
 			}
@@ -198,9 +211,14 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
 			if len(size) == len(magnetLinks) {
 				mySize = size[it]
 			}
+			if mySize == "" {
+				go func() {
+					_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
+				}()
+			}
 
-			ixt := IndexedTorrent{
-				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+			ixt := schema.IndexedTorrent{
+				Title:         releaseTitle,
 				OriginalTitle: title,
 				Details:       link,
 				Year:          year,
@@ -225,3 +243,15 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
 
 	return indexedTorrents, nil
 }
+
+func getPublishedDate(document *goquery.Document) time.Time {
+	var date time.Time
+	//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">
+	datePublished := strings.TrimSpace(document.Find("meta[property=\"article:published_time\"]").AttrOr("content", ""))
+
+	if datePublished != "" {
+		date, _ = time.Parse(time.RFC3339, datePublished)
+	}
+
+	return date
+}

api/comando_torrents.go

@@ -1,11 +1,9 @@
 package handler
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -16,11 +14,14 @@ import (
 	"github.com/felipemarinho97/torrent-indexer/magnet"
 	"github.com/felipemarinho97/torrent-indexer/schema"
 	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
 )
 
 var comando = IndexerMeta{
-	URL:       "https://comando.la/",
-	SearchURL: "?s=",
+	Label:       "comando",
+	URL:         "https://comando.la/",
+	SearchURL:   "?s=",
+	PagePattern: "page/%s",
 }
 
 var replacer = strings.NewReplacer(
@@ -39,30 +40,49 @@ var replacer = strings.NewReplacer(
 )
 
 func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	metadata := comando
+
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
+	}()
+
 	ctx := r.Context()
-	// supported query params: q, season, episode
+	// supported query params: q, season, episode, page, filter_results
 	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
 
 	// URL encode query param
 	q = url.QueryEscape(q)
-	url := comando.URL
+	url := metadata.URL
 	if q != "" {
-		url = fmt.Sprintf("%s%s%s", url, comando.SearchURL, q)
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
 	}
 
 	fmt.Println("URL:>", url)
-	resp, err := http.Get(url)
+	resp, err := i.requester.GetDocument(ctx, url)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
-		json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
-	defer resp.Body.Close()
+	defer resp.Close()
 
-	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	doc, err := goquery.NewDocumentFromReader(resp)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
-		json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
 		return
 	}
 
@@ -73,38 +93,29 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
 		links = append(links, link)
 	})
 
-	var itChan = make(chan []IndexedTorrent)
-	var errChan = make(chan error)
-	indexedTorrents := []IndexedTorrent{}
-	for _, link := range links {
-		go func(link string) {
-			torrents, err := getTorrents(ctx, i, link)
-			if err != nil {
-				fmt.Println(err)
-				errChan <- err
-			}
-			itChan <- torrents
-		}(link)
-	}
+	// extract each torrent link
+	indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrents(ctx, i, link)
+	})
 
-	for i := 0; i < len(links); i++ {
-		select {
-		case torrents := <-itChan:
-			indexedTorrents = append(indexedTorrents, torrents...)
-		case err := <-errChan:
-			fmt.Println(err)
-		}
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
 	}
 
 	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(Response{
-		Results: indexedTorrents,
-		Count:   len(indexedTorrents),
+	err = json.NewEncoder(w).Encode(Response{
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
 	})
+	if err != nil {
+		fmt.Println(err)
+	}
 }
 
-func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
-	var indexedTorrents []IndexedTorrent
+func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+	var indexedTorrents []schema.IndexedTorrent
 	doc, err := getDocument(ctx, i, link)
 	if err != nil {
 		return nil, err
@@ -116,19 +127,11 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
 	// div itemprop="datePublished"
 	datePublished := strings.TrimSpace(article.Find("div[itemprop=\"datePublished\"]").Text())
 	// pattern: 10 de setembro de 2021
-	re := regexp.MustCompile(`(\d{2}) de (\w+) de (\d{4})`)
-	matches := re.FindStringSubmatch(datePublished)
-	var date time.Time
-	if len(matches) > 0 {
-		day := matches[1]
-		month := matches[2]
-		year := matches[3]
-		datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
-		date, err = time.Parse("2006-01-02", datePublished)
-		if err != nil {
-			return nil, err
-		}
+	date, err := parseLocalizedDate(datePublished)
+	if err != nil {
+		return nil, err
 	}
+
 	magnets := textContent.Find("a[href^=\"magnet\"]")
 	var magnetLinks []string
 	magnets.Each(func(i int, s *goquery.Selection) {
@@ -159,24 +162,26 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
 		text := s.Text()
 
 		audio = append(audio, findAudioFromText(text)...)
-		year = findYearFromText(text, title)
+		y := findYearFromText(text, title)
+		if y != "" {
+			year = y
+		}
 		size = append(size, findSizesFromText(text)...)
 	})
 
 	// find any link from imdb
 	imdbLink := ""
-	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
+	article.Find("a").Each(func(i int, s *goquery.Selection) {
 		link, _ := s.Attr("href")
-		re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
-		matches := re.FindStringSubmatch(link)
-		if len(matches) > 0 {
-			imdbLink = matches[0]
+		_imdbLink, err := getIMDBLink(link)
+		if err == nil {
+			imdbLink = _imdbLink
 		}
 	})
 
-	size = stableUniq(size)
+	size = utils.StableUniq(size)
 
-	var chanIndexedTorrent = make(chan IndexedTorrent)
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
 
 	// for each magnet link, create a new indexed torrent
 	for it, magnetLink := range magnetLinks {
@@ -189,21 +194,9 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
 			releaseTitle := magnet.DisplayName
 			infoHash := magnet.InfoHash.String()
 			trackers := magnet.Trackers
-			magnetAudio := []schema.Audio{}
-			if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
-				magnetAudio = append(magnetAudio, audio...)
-			} else if len(audio) > 1 {
-				// remove portuguese audio, and append to magnetAudio
-				for _, a := range audio {
-					if a != schema.AudioPortuguese {
-						magnetAudio = append(magnetAudio, a)
-					}
-				}
-			} else {
-				magnetAudio = append(magnetAudio, audio...)
-			}
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
 
-			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, infoHash, trackers)
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
 			if err != nil {
 				fmt.Println(err)
 			}
@@ -215,9 +208,14 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
 			if len(size) == len(magnetLinks) {
 				mySize = size[it]
 			}
+			if mySize == "" {
+				go func() {
+					_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
+				}()
+			}
 
-			ixt := IndexedTorrent{
-				Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+			ixt := schema.IndexedTorrent{
+				Title:         releaseTitle,
 				OriginalTitle: title,
 				Details:       link,
 				Year:          year,
@@ -243,86 +241,25 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
 	return indexedTorrents, nil
 }
 
-func stableUniq(s []string) []string {
-	var uniq []map[string]interface{}
-	m := make(map[string]map[string]interface{})
-	for i, v := range s {
-		m[v] = map[string]interface{}{
-			"v": v,
-			"i": i,
+func parseLocalizedDate(datePublished string) (time.Time, error) {
+	re := regexp.MustCompile(`(\d{1,2}) de (\w+) de (\d{4})`)
+	matches := re.FindStringSubmatch(datePublished)
+	if len(matches) > 0 {
+		day := matches[1]
+		// append 0 to single digit day
+		if len(day) == 1 {
+			day = fmt.Sprintf("0%s", day)
 		}
-	}
-	// to order by index
-	for _, v := range m {
-		uniq = append(uniq, v)
-	}
-
-	// sort by index
-	for i := 0; i < len(uniq); i++ {
-		for j := i + 1; j < len(uniq); j++ {
-			if uniq[i]["i"].(int) > uniq[j]["i"].(int) {
-				uniq[i], uniq[j] = uniq[j], uniq[i]
-			}
+		month := matches[2]
+		year := matches[3]
+		datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
+		date, err := time.Parse("2006-01-02", datePublished)
+		if err != nil {
+			return time.Time{}, err
 		}
+		return date, nil
 	}
-
-	// get only values
-	var uniqValues []string
-	for _, v := range uniq {
-		uniqValues = append(uniqValues, v["v"].(string))
-	}
-
-	return uniqValues
-}
-
-func findYearFromText(text string, title string) (year string) {
-	re := regexp.MustCompile(`Lançamento: (.*)`)
-	yearMatch := re.FindStringSubmatch(text)
-	if len(yearMatch) > 0 {
-		year = yearMatch[1]
-	}
-
-	if year == "" {
-		re = regexp.MustCompile(`\((\d{4})\)`)
-		yearMatch := re.FindStringSubmatch(title)
-		if len(yearMatch) > 0 {
-			year = yearMatch[1]
-		}
-	}
-	return year
-}
-
-func findAudioFromText(text string) []schema.Audio {
-	var audio []schema.Audio
-	re := regexp.MustCompile(`(.udio|Idioma):.?(.*)`)
-	audioMatch := re.FindStringSubmatch(text)
-	if len(audioMatch) > 0 {
-		sep := getSeparator(audioMatch[2])
-		langs_raw := strings.Split(audioMatch[2], sep)
-		for _, lang := range langs_raw {
-			lang = strings.TrimSpace(lang)
-			a := schema.GetAudioFromString(lang)
-			if a != nil {
-				audio = append(audio, *a)
-			} else {
-				fmt.Println("unknown language:", lang)
-			}
-		}
-	}
-	return audio
-}
-
-func findSizesFromText(text string) []string {
-	var sizes []string
-	// everything that ends with GB or MB, using ',' or '.' as decimal separator
-	re := regexp.MustCompile(`(\d+[\.,]?\d+) ?(GB|MB)`)
-	sizesMatch := re.FindAllStringSubmatch(text, -1)
-	if len(sizesMatch) > 0 {
-		for _, size := range sizesMatch {
-			sizes = append(sizes, size[0])
-		}
-	}
-	return sizes
+	return time.Time{}, nil
 }
 
 func processTitle(title string, a []schema.Audio) string {
@@ -337,55 +274,3 @@ func processTitle(title string, a []schema.Audio) string {
 
 	return title
 }
-
-func appendAudioISO639_2Code(title string, a []schema.Audio) string {
-	if len(a) > 0 {
-		audio := []string{}
-		for _, lang := range a {
-			audio = append(audio, lang.String())
-		}
-		title = fmt.Sprintf("%s (%s)", title, strings.Join(audio, ", "))
-	}
-	return title
-}
-
-func getSeparator(s string) string {
-	if strings.Contains(s, "|") {
-		return "|"
-	} else if strings.Contains(s, ",") {
-		return ","
-	}
-	return " "
-}
-
-func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Document, error) {
-	// try to get from redis first
-	docCache, err := i.redis.Get(ctx, link)
-	if err == nil {
-		return goquery.NewDocumentFromReader(ioutil.NopCloser(bytes.NewReader(docCache)))
-	}
-
-	resp, err := http.Get(link)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	// set cache
-	err = i.redis.Set(ctx, link, body)
-	if err != nil {
-		fmt.Println(err)
-	}
-
-	doc, err := goquery.NewDocumentFromReader(ioutil.NopCloser(bytes.NewReader(body)))
-	if err != nil {
-		return nil, err
-	}
-
-	return doc, nil
-}

api/comando_torrents_test.go

@@ -3,76 +3,45 @@ package handler
 import (
 	"reflect"
 	"testing"
-
-	"github.com/felipemarinho97/torrent-indexer/schema"
+	"time"
 )
 
-func Test_findAudioFromText(t *testing.T) {
+func Test_parseLocalizedDate(t *testing.T) {
 	type args struct {
-		text string
+		datePublished string
 	}
 	tests := []struct {
-		name string
-		args args
-		want []schema.Audio
+		name    string
+		args    args
+		want    time.Time
+		wantErr bool
 	}{
 		{
-			name: "should return audio in portuguese",
+			name: "should return date",
 			args: args{
-				text: "Áudio: Português",
-			},
-			want: []schema.Audio{
-				schema.AudioPortuguese,
+				datePublished: "12 de outubro de 2022",
 			},
+			want:    time.Date(2022, 10, 12, 0, 0, 0, 0, time.UTC),
+			wantErr: false,
 		},
 		{
-			name: "should return audio in portuguese",
+			name: "should return date single digit",
 			args: args{
-				text: "Idioma: Português",
-			},
-			want: []schema.Audio{
-				schema.AudioPortuguese,
-			},
-		},
-		{
-			name: "should return audio in portuguese",
-			args: args{
-				text: "Audio: Português",
-			},
-			want: []schema.Audio{
-				schema.AudioPortuguese,
-			},
-		},
-		{
-			name: "should return audio in portuguese",
-			args: args{
-				text: `
-»INFORMAÇÕES«
-Título Traduzido: O Cangaceiro do Futuro
-Título Original: O Cangaceiro do Futuro
-IMDb: 7,1
-Gênero:Comédia
-Lançamento: 2022
-Qualidade: WEB-DL
-Áudio: Português
-Legenda: S/L
-Formato: MKV
-Tamanho: 5.77 GB | 9.60 GB
-Duração: 30 Min./Ep.
-Qualidade de Áudio: 10
-Qualidade de Vídeo: 10
-Servidor Via: Torrent
-				`,
-			},
-			want: []schema.Audio{
-				schema.AudioPortuguese,
+				datePublished: "1 de outubro de 2022",
 			},
+			want:    time.Date(2022, 10, 1, 0, 0, 0, 0, time.UTC),
+			wantErr: false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := findAudioFromText(tt.args.text); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("findAudioFromText() = %v, want %v", got, tt.want)
+			got, err := parseLocalizedDate(tt.args.datePublished)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseDate() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseDate() = %v, want %v", got, tt.want)
 			}
 		})
 	}

api/comandohds.go

@@ -0,0 +1,226 @@
+package handler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+var comandohds = IndexerMeta{
+	Label:       "comandohds",
+	URL:         "https://comandohds.org/",
+	SearchURL:   "?s=",
+	PagePattern: "page/%s",
+}
+
+var title_re = regexp.MustCompile(`^[(Filme)|(Série)\s]+`)
+
+func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	metadata := comandohds
+
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
+	}()
+
+	ctx := r.Context()
+	// supported query params: q, page, filter_results
+	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
+
+	// URL encode query param
+	q = url.QueryEscape(q)
+	url := metadata.URL
+	if q != "" {
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
+	}
+
+	fmt.Println("URL:>", url)
+	resp, err := i.requester.GetDocument(ctx, url)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
+		return
+	}
+	defer resp.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
+		return
+	}
+
+	var links []string
+	doc.Find(".post").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Find("div.title > a").Attr("href")
+		links = append(links, link)
+	})
+
+	// extract each torrent link
+	indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrentsComandoHDs(ctx, i, link)
+	})
+
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
+func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+	var indexedTorrents []schema.IndexedTorrent
+	doc, err := getDocument(ctx, i, link)
+	if err != nil {
+		return nil, err
+	}
+
+	article := doc.Find("article")
+	title := title_re.ReplaceAllString(article.Find(".main_title > h1").Text(), "")
+	textContent := article.Find("div.content")
+	date := getPublishedDateFromMeta(doc)
+	magnets := textContent.Find("a[href^=\"magnet\"]")
+	var magnetLinks []string
+	magnets.Each(func(i int, s *goquery.Selection) {
+		magnetLink, _ := s.Attr("href")
+		magnetLinks = append(magnetLinks, magnetLink)
+	})
+
+	var audio []schema.Audio
+	var year string
+	var size []string
+	article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
+		// pattern:
+		// »INFORMAÇÕES«
+		// Titulo Traduzido: O Guerreiro Banido
+		// Titulo Original: 天龍八部之喬峰傳
+		// <picture />: 5.7
+		// Ano de Lançamento: 2023
+		// Gênero: Ação
+		// Formato: MKV
+		// Qualidade: WEB-DL
+		// Idioma: Português | Inglês
+		// Legenda: Português
+		// Tamanho: – GB
+		// Qualidade Áudio e Vídeo: 10
+		// Duração: 130 Min
+		// Servidor: Torrent
+		text := s.Text()
+
+		audio = append(audio, findAudioFromText(text)...)
+		y := findYearFromText(text, title)
+		if y != "" {
+			year = y
+		}
+		size = append(size, findSizesFromText(text)...)
+	})
+
+	// find any link from imdb
+	imdbLink := ""
+	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Attr("href")
+		_imdbLink, err := getIMDBLink(link)
+		if err == nil {
+			imdbLink = _imdbLink
+		}
+	})
+
+	size = utils.StableUniq(size)
+
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
+
+	// for each magnet link, create a new indexed torrent
+	for it, magnetLink := range magnetLinks {
+		it := it
+		go func(it int, magnetLink string) {
+			magnet, err := magnet.ParseMagnetUri(magnetLink)
+			if err != nil {
+				fmt.Println(err)
+			}
+			releaseTitle := strings.TrimSpace(magnet.DisplayName)
+			infoHash := magnet.InfoHash.String()
+			trackers := magnet.Trackers
+			for i, tracker := range trackers {
+				trackers[i] = strings.TrimSpace(tracker)
+			}
+
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
+
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+			if err != nil {
+				fmt.Println(err)
+			}
+
+			title := processTitle(title, magnetAudio)
+
+			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
+			var mySize string
+			if len(size) == len(magnetLinks) {
+				mySize = size[it]
+			}
+			if mySize == "" {
+				go func() {
+					_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
+				}()
+			}
+
+			ixt := schema.IndexedTorrent{
+				Title:         releaseTitle,
+				OriginalTitle: title,
+				Details:       link,
+				Year:          year,
+				IMDB:          imdbLink,
+				Audio:         magnetAudio,
+				MagnetLink:    magnetLink,
+				Date:          date,
+				InfoHash:      infoHash,
+				Trackers:      trackers,
+				LeechCount:    peer,
+				SeedCount:     seed,
+				Size:          mySize,
+			}
+			chanIndexedTorrent <- ixt
+		}(it, magnetLink)
+	}
+
+	for i := 0; i < len(magnetLinks); i++ {
+		it := <-chanIndexedTorrent
+		indexedTorrents = append(indexedTorrents, it)
+	}
+
+	return indexedTorrents, nil
+}

api/common.go

@@ -0,0 +1,227 @@
+package handler
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"regexp"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+)
+
+// getDocument retrieves a document from the cache or makes a request to get it.
+// It first checks the Redis cache for the document body.
+func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Document, error) {
+	// try to get from redis first
+	docCache, err := i.redis.Get(ctx, link)
+	if err == nil {
+		i.metrics.CacheHits.WithLabelValues("document_body").Inc()
+		fmt.Printf("returning from long-lived cache: %s\n", link)
+		return goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(docCache)))
+	}
+	defer i.metrics.CacheMisses.WithLabelValues("document_body").Inc()
+
+	resp, err := i.requester.GetDocument(ctx, link)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Close()
+
+	body, err := io.ReadAll(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	// set cache
+	err = i.redis.Set(ctx, link, body)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(body)))
+	if err != nil {
+		return nil, err
+	}
+
+	return doc, nil
+}
+
+func getPublishedDateFromMeta(document *goquery.Document) time.Time {
+	var date time.Time
+	//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">
+	datePublished := strings.TrimSpace(document.Find("meta[property=\"article:published_time\"]").AttrOr("content", ""))
+
+	if datePublished != "" {
+		date, _ = time.Parse(time.RFC3339, datePublished)
+	}
+
+	return date
+}
+
+type datePattern struct {
+	regex  *regexp.Regexp
+	layout string
+}
+
+var datePatterns = []datePattern{
+	{regexp.MustCompile(`\d{4}-\d{2}-\d{2}`), "2006-01-02"},
+	{regexp.MustCompile(`\d{2}-\d{2}-\d{4}`), "02-01-2006"},
+	{regexp.MustCompile(`\d{2}/\d{2}/\d{4}`), "02/01/2006"},
+}
+
+// getPublishedDateFromRawString extracts the date from a raw string using predefined patterns.
+func getPublishedDateFromRawString(dateStr string) time.Time {
+	for _, p := range datePatterns {
+		match := p.regex.FindString(dateStr)
+
+		if match != "" {
+			date, err := time.Parse(p.layout, match)
+			if err == nil {
+				return date.UTC()
+			}
+		}
+	}
+
+	return time.Time{}
+}
+
+// getSeparator returns the separator used in the string.
+// It checks for common separators like "|", ",", "/", and " e "
+func getSeparator(s string) string {
+	if strings.Contains(s, "|") {
+		return "|"
+	} else if strings.Contains(s, ",") {
+		return ","
+	} else if strings.Contains(s, "/") {
+		return "/"
+	} else if strings.Contains(s, " e ") {
+		return " e "
+	}
+	return " "
+}
+
+// findAudioFromText extracts audio languages from a given text.
+// It looks for patterns like "Áudio: Português, Inglês" or "Idioma: Português, Inglês"
+func findAudioFromText(text string) []schema.Audio {
+	var audio []schema.Audio
+	re := regexp.MustCompile(`(.udio|Idioma):.?(.*)`)
+	audioMatch := re.FindStringSubmatch(text)
+	if len(audioMatch) > 0 {
+		sep := getSeparator(audioMatch[2])
+		langs_raw := strings.Split(audioMatch[2], sep)
+		for _, lang := range langs_raw {
+			lang = strings.TrimSpace(lang)
+			a := schema.GetAudioFromString(lang)
+			if a != nil {
+				audio = append(audio, *a)
+			} else {
+				fmt.Println("unknown language:", lang)
+			}
+		}
+	}
+	return audio
+}
+
+// findYearFromText extracts the year from a given text.
+// It looks for patterns like "Lançamento: 2001" in the title.
+func findYearFromText(text string, title string) (year string) {
+	re := regexp.MustCompile(`Lançamento: (.*)`)
+	yearMatch := re.FindStringSubmatch(text)
+	if len(yearMatch) > 0 {
+		year = yearMatch[1]
+	}
+
+	if year == "" {
+		re = regexp.MustCompile(`\((\d{4})\)`)
+		yearMatch := re.FindStringSubmatch(title)
+		if len(yearMatch) > 0 {
+			year = yearMatch[1]
+		}
+	}
+	return strings.TrimSpace(year)
+}
+
+// findSizesFromText extracts sizes from a given text.
+// It looks for patterns like "Tamanho: 1.26 GB" or "Tamanho: 700 MB".
+func findSizesFromText(text string) []string {
+	var sizes []string
+	// everything that ends with GB or MB, using ',' or '.' as decimal separator
+	re := regexp.MustCompile(`(\d+[\.,]?\d+) ?(GB|MB)`)
+	sizesMatch := re.FindAllStringSubmatch(text, -1)
+	if len(sizesMatch) > 0 {
+		for _, size := range sizesMatch {
+			sizes = append(sizes, size[0])
+		}
+	}
+	return sizes
+}
+
+// getIMDBLink extracts the IMDB link from a given link.
+// It looks for patterns like "https://www.imdb.com/title/tt1234567/".
+// Returns an error if no valid IMDB link is found.
+func getIMDBLink(link string) (string, error) {
+	var imdbLink string
+	re := regexp.MustCompile(`https://www.imdb.com(/[a-z]{2})?/title/(tt\d+)/?`)
+
+	matches := re.FindStringSubmatch(link)
+	if len(matches) > 0 {
+		imdbLink = matches[0]
+	} else {
+		return "", fmt.Errorf("no imdb link found")
+	}
+	return imdbLink, nil
+}
+
+// appendAudioISO639_2Code appends the audio languages to the title in ISO 639-2 code format.
+// It formats the title to include the audio languages in parentheses.
+// Example: "Movie Title (eng, por)"
+func appendAudioISO639_2Code(title string, a []schema.Audio) string {
+	if len(a) > 0 {
+		audio := []string{}
+		for _, lang := range a {
+			audio = append(audio, lang.String())
+		}
+		title = fmt.Sprintf("%s (%s)", title, strings.Join(audio, ", "))
+	}
+	return title
+}
+
+// getAudioFromTitle extracts audio languages from the release title.
+// It checks for common patterns like "nacional", "dual", or "dublado"
+func getAudioFromTitle(releaseTitle string, audioFromContent []schema.Audio) []schema.Audio {
+	magnetAudio := []schema.Audio{}
+	isNacional := strings.Contains(strings.ToLower(releaseTitle), "nacional")
+	if isNacional {
+		magnetAudio = append(magnetAudio, schema.AudioPortuguese)
+	}
+
+	if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
+		magnetAudio = append(magnetAudio, audioFromContent...)
+		// if Portuguese audio is not in the audio slice, append it
+		if !slices.Contains(magnetAudio, schema.AudioPortuguese) {
+			magnetAudio = append(magnetAudio, schema.AudioPortuguese)
+		}
+	} else if len(audioFromContent) > 1 {
+		// remove portuguese audio, and append to magnetAudio
+		for _, a := range audioFromContent {
+			if a != schema.AudioPortuguese {
+				magnetAudio = append(magnetAudio, a)
+			}
+		}
+	} else {
+		magnetAudio = append(magnetAudio, audioFromContent...)
+	}
+
+	// order and uniq the audio slice
+	slices.SortFunc(magnetAudio, func(a, b schema.Audio) int {
+		return strings.Compare(a.String(), b.String())
+	})
+	magnetAudio = slices.Compact(magnetAudio)
+
+	return magnetAudio
+}

api/common_test.go

@@ -0,0 +1,241 @@
+package handler
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/felipemarinho97/torrent-indexer/schema"
+)
+
+func Test_getPublishedDateFromRawString(t *testing.T) {
+	type args struct {
+		dateStr string
+	}
+	tests := []struct {
+		name string
+		args args
+		want time.Time
+	}{
+		{
+			name: "should parse date in format 2025-01-01",
+			args: args{
+				dateStr: "2025-01-01",
+			},
+			want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "should parse date in format 01-01-2025",
+			args: args{
+				dateStr: "01-01-2025",
+			},
+			want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "should parse date in format 01/01/2025",
+			args: args{
+				dateStr: "01/01/2025",
+			},
+			want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
+		},
+		{
+			name: "should parse date from starck-filmes link",
+			args: args{
+				dateStr: "https://www.starckfilmes.online/catalog/jogos-de-seducao-2025-18-07-2025/",
+			},
+			want: time.Date(2025, 7, 18, 0, 0, 0, 0, time.UTC),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getPublishedDateFromRawString(tt.args.dateStr); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("getPublishedDateFromRawString() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_findAudioFromText(t *testing.T) {
+	type args struct {
+		text string
+	}
+	tests := []struct {
+		name string
+		args args
+		want []schema.Audio
+	}{
+		{
+			name: "should return audio in portuguese",
+			args: args{
+				text: "Áudio: Português",
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+			},
+		},
+		{
+			name: "should return audio in portuguese",
+			args: args{
+				text: "Idioma: Português",
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+			},
+		},
+		{
+			name: "should return audio in portuguese",
+			args: args{
+				text: "Audio: Português",
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+			},
+		},
+		{
+			name: "should return audio in portuguese - comando_torrents",
+			args: args{
+				text: `
+»INFORMAÇÕES«
+Título Traduzido: O Cangaceiro do Futuro
+Título Original: O Cangaceiro do Futuro
+IMDb: 7,1
+Gênero:Comédia
+Lançamento: 2022
+Qualidade: WEB-DL
+Áudio: Português
+Legenda: S/L
+Formato: MKV
+Tamanho: 5.77 GB | 9.60 GB
+Duração: 30 Min./Ep.
+Qualidade de Áudio: 10
+Qualidade de Vídeo: 10
+Servidor Via: Torrent
+				`,
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+			},
+		},
+		{
+			name: "should return audio in portuguese - rede torrent",
+			args: args{
+				text: `
+Filme Bicho de Sete Cabeças Torrent
+Título Original: Bicho de Sete Cabeças
+Lançamento: 2001
+Gêneros: Drama / Nacional
+Idioma: Português
+Qualidade: 720p / BluRay
+Duração: 1h 14 Minutos
+Formato: Mp4
+Vídeo: 10 e Áudio: 10
+Legendas: Português
+Nota do Imdb: 7.7
+Tamanho: 1.26 GB
+				`,
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+			},
+		},
+		{
+			name: "should return audio in portuguese - rede torrent 2",
+			args: args{
+				text: `
+Filme Branca de Neve e o Caçador Torrent / Assistir Online
+Título Original: Snow White and the Huntsman
+Lançamento: 2012
+Gêneros: Ação / Aventura / Fantasia
+Idioma: Português / Inglês
+Duração: 126 Minutos
+Formato: Mkv / Mp4
+Vídeo: 10 e Áudio: 10
+Legendas: Sim
+Tamanho: 2.69 GB / 1.95 GB / 1.0 GB
+				`,
+			},
+			want: []schema.Audio{
+				schema.AudioPortuguese,
+				schema.AudioEnglish,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := findAudioFromText(tt.args.text); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("findAudioFromText() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_getIMDBLink(t *testing.T) {
+	type args struct {
+		link string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "should return imdb link",
+			args: args{
+				link: "https://www.imdb.com/title/tt1234567",
+			},
+			want:    "https://www.imdb.com/title/tt1234567",
+			wantErr: false,
+		},
+		{
+			name: "should return imdb link when end with /",
+			args: args{
+				link: "https://www.imdb.com/title/tt1234567/",
+			},
+			want:    "https://www.imdb.com/title/tt1234567/",
+			wantErr: false,
+		},
+		{
+			name: "should return imdb link when end with /",
+			args: args{
+				link: "https://www.imdb.com/title/tt1234567/",
+			},
+			want:    "https://www.imdb.com/title/tt1234567/",
+			wantErr: false,
+		},
+		{
+			name: "should return imdb link when it has a language",
+			args: args{
+				link: "https://www.imdb.com/pt/title/tt18722864/",
+			},
+			want: "https://www.imdb.com/pt/title/tt18722864/",
+		},
+		{
+			name: "should return imdb link when it has a language",
+			args: args{
+				link: "https://www.imdb.com/pt/title/tt34608980/",
+			},
+			want: "https://www.imdb.com/pt/title/tt34608980/",
+		},
+		{
+			name: "should return error when link is invalid",
+			args: args{
+				link: "https://www.google.com",
+			},
+			want:    "",
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := getIMDBLink(tt.args.link)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("getIMDBLink() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("getIMDBLink() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

api/index.go

@@ -6,42 +6,59 @@ import (
 	"time"
 
 	"github.com/felipemarinho97/torrent-indexer/cache"
+	"github.com/felipemarinho97/torrent-indexer/consts"
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/monitoring"
+	"github.com/felipemarinho97/torrent-indexer/requester"
 	"github.com/felipemarinho97/torrent-indexer/schema"
+	meilisearch "github.com/felipemarinho97/torrent-indexer/search"
 )
 
 type Indexer struct {
-	redis *cache.Redis
+	redis             *cache.Redis
+	metrics           *monitoring.Metrics
+	requester         *requester.Requster
+	search            *meilisearch.SearchIndexer
+	magnetMetadataAPI *magnet.MetadataClient
+	postProcessors    []PostProcessorFunc
 }
 
 type IndexerMeta struct {
-	URL       string
-	SearchURL string
+	Label       string // Label is used for Prometheus metrics and logging. Must be alphanumeric optionally with underscores.
+	URL         string // URL is the base URL of the indexer, e.g. "https://example.com/"
+	SearchURL   string // SearchURL is the base URL for search queries, e.g. "?s="
+	PagePattern string // PagePattern for pagination, e.g. "page/%s"
 }
 
 type Response struct {
-	Results []IndexedTorrent `json:"results"`
-	Count   int              `json:"count"`
+	Results []schema.IndexedTorrent `json:"results"`
+	Count   int                     `json:"count"`
 }
 
-type IndexedTorrent struct {
-	Title         string         `json:"title"`
-	OriginalTitle string         `json:"original_title"`
-	Details       string         `json:"details"`
-	Year          string         `json:"year"`
-	IMDB          string         `json:"imdb"`
-	Audio         []schema.Audio `json:"audio"`
-	MagnetLink    string         `json:"magnet_link"`
-	Date          time.Time      `json:"date"`
-	InfoHash      string         `json:"info_hash"`
-	Trackers      []string       `json:"trackers"`
-	Size          string         `json:"size"`
-	LeechCount    int            `json:"leech_count"`
-	SeedCount     int            `json:"seed_count"`
+type PostProcessorFunc func(*Indexer, *http.Request, []schema.IndexedTorrent) []schema.IndexedTorrent
+
+var GlobalPostProcessors = []PostProcessorFunc{
+	AddSimilarityCheck,     // Jaccard similarity
+	FullfilMissingMetadata, // Fill missing size or title metadata
+	CleanupTitleWebsites,   // Remove website names from titles
+	AppendAudioTags,        // Add (brazilian, eng, etc.) audio tags to titles
+	SendToSearchIndexer,    // Send indexed torrents to Meilisearch
 }
 
-func NewIndexers(redis *cache.Redis) *Indexer {
+func NewIndexers(
+	redis *cache.Redis,
+	metrics *monitoring.Metrics,
+	req *requester.Requster,
+	si *meilisearch.SearchIndexer,
+	mc *magnet.MetadataClient,
+) *Indexer {
 	return &Indexer{
-		redis: redis,
+		redis:             redis,
+		metrics:           metrics,
+		requester:         req,
+		search:            si,
+		magnetMetadataAPI: mc,
+		postProcessors:    GlobalPostProcessors,
 	}
 }
 
@@ -49,23 +66,105 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
 	currentTime := time.Now().Format(time.RFC850)
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)
-	json.NewEncoder(w).Encode(map[string]interface{}{
-		"time": currentTime,
+	err := json.NewEncoder(w).Encode(map[string]interface{}{
+		"time":  currentTime,
+		"build": consts.GetBuildInfo(),
 		"endpoints": map[string]interface{}{
-			"/indexers/comando_torrents": map[string]interface{}{
-				"method":      "GET",
-				"description": "Indexer for comando torrents",
-				"query_params": map[string]string{
-					"q": "search query",
+			"/indexers/comando_torrents": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"description": "Indexer for comando torrents",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"page":           "page number",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
 				},
 			},
-			"/indexers/bludv": map[string]interface{}{
-				"method":      "GET",
-				"description": "Indexer for bludv",
-				"query_params": map[string]string{
-					"q": "search query",
+			"/indexers/bludv": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"description": "Indexer for bludv",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"page":           "page number",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					}},
+			},
+			"/indexers/torrent-dos-filmes": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"page":        "page number",
+					"description": "Indexer for Torrent dos Filmes",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
+				},
+			},
+			"/indexers/comandohds": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"page":        "page number",
+					"description": "Indexer for Comando HDs",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
+				},
+			},
+			"/indexers/starck-filmes": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"page":        "page number",
+					"description": "Indexer for Starck Filmes",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
+				},
+			},
+			"/indexers/rede_torrent": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"description": "Indexer for rede torrent",
+					"query_params": map[string]string{
+						"q":              "search query",
+						"page":           "page number",
+						"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
+					},
+				},
+			},
+			"/indexers/manual": []map[string]interface{}{
+				{
+					"method":      "POST",
+					"description": "Add a manual torrent entry to the indexer for 12 hours",
+					"body": map[string]interface{}{
+						"magnetLink": "magnet link",
+					}},
+				{
+					"method":      "GET",
+					"description": "Get all manual torrents",
+				},
+			},
+			"/search": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"description": "Search for cached torrents across all indexers",
+					"query_params": map[string]string{
+						"q": "search query",
+					},
+				},
+			},
+			"/ui/": []map[string]interface{}{
+				{
+					"method":      "GET",
+					"description": "Show the unified search UI (only work if Meilisearch is enabled)",
 				},
 			},
 		},
 	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
 }

api/manual.go

@@ -0,0 +1,125 @@
+package handler
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/redis/go-redis/v9"
+)
+
+const manualTorrentsRedisKey = "manual:torrents"
+
+var manualTorrentExpiration = 8 * time.Hour
+
+type ManualIndexerRequest struct {
+	MagnetLink string `json:"magnetLink"`
+}
+
+func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	var req ManualIndexerRequest
+	indexedTorrents := []schema.IndexedTorrent{}
+
+	// fetch from redis
+	out, err := i.redis.Get(ctx, manualTorrentsRedisKey)
+	if err != nil && !errors.Is(err, redis.Nil) {
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Println(err)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
+		return
+	} else if errors.Is(err, redis.Nil) {
+		out = bytes.NewBufferString("[]").Bytes()
+	}
+
+	err = json.Unmarshal([]byte(out), &indexedTorrents)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
+		return
+	}
+
+	// check if the request is a POST
+	if r.Method == http.MethodPost {
+		// decode the request body
+		err := json.NewDecoder(r.Body).Decode(&req)
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+			if err != nil {
+				fmt.Println(err)
+			}
+			i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
+			return
+		}
+
+		magnet, err := magnet.ParseMagnetUri(req.MagnetLink)
+		if err != nil {
+			fmt.Println(err)
+		}
+		var audio []schema.Audio
+		releaseTitle := magnet.DisplayName
+		infoHash := magnet.InfoHash.String()
+		trackers := magnet.Trackers
+		magnetAudio := getAudioFromTitle(releaseTitle, audio)
+
+		peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		title := processTitle(releaseTitle, magnetAudio)
+
+		ixt := schema.IndexedTorrent{
+			Title:         appendAudioISO639_2Code(releaseTitle, magnetAudio),
+			OriginalTitle: title,
+			Audio:         magnetAudio,
+			MagnetLink:    req.MagnetLink,
+			InfoHash:      infoHash,
+			Trackers:      trackers,
+			LeechCount:    peer,
+			SeedCount:     seed,
+		}
+
+		// write to redis
+		indexedTorrents = append(indexedTorrents, ixt)
+		out, err := json.Marshal(indexedTorrents)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		err = i.redis.SetWithExpiration(ctx, manualTorrentsRedisKey, out, manualTorrentExpiration)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+			if err != nil {
+				fmt.Println(err)
+			}
+			i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
+			return
+		}
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: indexedTorrents,
+		Count:   len(indexedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}

api/post_processors.go

@@ -0,0 +1,100 @@
+package handler
+
+import (
+	"fmt"
+	"net/http"
+	"slices"
+	"strings"
+
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+	"github.com/hbollon/go-edlib"
+)
+
+// CleanupTitleWebsites removes unwanted characters from the title
+func CleanupTitleWebsites(_ *Indexer, _ *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
+	for i := range torrents {
+		torrents[i].Title = utils.RemoveKnownWebsites(torrents[i].Title)
+	}
+	return torrents
+}
+
+func AppendAudioTags(_ *Indexer, _ *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
+	for i, it := range torrents {
+		torrents[i].Title = appendAudioISO639_2Code(torrents[i].Title, it.Audio)
+	}
+
+	return torrents
+}
+
+// SendToSearchIndexer sends the indexed torrents to the search indexer
+func SendToSearchIndexer(i *Indexer, _ *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
+	go func() {
+		_ = i.search.IndexTorrents(torrents)
+	}()
+	return torrents
+}
+
+// FullfilMissingMetadata fills in missing metadata for indexed torrents
+func FullfilMissingMetadata(i *Indexer, r *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
+	if !i.magnetMetadataAPI.IsEnabled() {
+		return torrents
+	}
+
+	return utils.ParallelFlatMap(torrents, func(it schema.IndexedTorrent) ([]schema.IndexedTorrent, error) {
+		if it.Size != "" && it.Title != "" && it.OriginalTitle != "" {
+			return []schema.IndexedTorrent{it}, nil
+		}
+		m, err := i.magnetMetadataAPI.FetchMetadata(r.Context(), it.MagnetLink)
+		if err != nil {
+			return []schema.IndexedTorrent{it}, nil
+		}
+
+		// convert size in bytes to a human-readable format
+		it.Size = utils.FormatBytes(m.Size)
+
+		// Use name from metadata if available as it is more accurate
+		if m.Name != "" {
+			it.Title = m.Name
+		}
+		fmt.Printf("hash: %s get -> size: %s\n", m.InfoHash, it.Size)
+
+		// If files are present, add them to the indexed torrent
+		if len(m.Files) > 0 {
+			it.Files = make([]schema.File, len(m.Files))
+			for i, file := range m.Files {
+				it.Files[i] = schema.File{
+					Path: file.Path,
+					Size: utils.FormatBytes(file.Size),
+				}
+			}
+		}
+
+		return []schema.IndexedTorrent{it}, nil
+	})
+}
+
+func AddSimilarityCheck(i *Indexer, r *http.Request, torrents []schema.IndexedTorrent) []schema.IndexedTorrent {
+	q := r.URL.Query().Get("q")
+
+	for i, it := range torrents {
+		jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
+		qLower := strings.ToLower(q)
+		splitLength := 2
+		torrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
+	}
+
+	// remove the ones with zero similarity
+	if len(torrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
+		torrents = utils.Filter(torrents, func(it schema.IndexedTorrent) bool {
+			return it.Similarity > 0
+		})
+	}
+
+	// sort by similarity
+	slices.SortFunc(torrents, func(i, j schema.IndexedTorrent) int {
+		return int((j.Similarity - i.Similarity) * 1000)
+	})
+
+	return torrents
+}

api/rede_torrent.go

@@ -0,0 +1,250 @@
+package handler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+var rede_torrent = IndexerMeta{
+	Label:       "rede_torrent",
+	URL:         "https://redetorrent.com/",
+	SearchURL:   "index.php?s=",
+	PagePattern: "%s",
+}
+
+func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	metadata := rede_torrent
+
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
+	}()
+
+	ctx := r.Context()
+	// supported query params: q, season, episode, page, filter_results
+	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
+
+	// URL encode query param
+	q = url.QueryEscape(q)
+	url := metadata.URL
+	if q != "" {
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
+	}
+
+	fmt.Println("URL:>", url)
+	resp, err := i.requester.GetDocument(ctx, url)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
+		return
+	}
+	defer resp.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
+		return
+	}
+
+	var links []string
+	doc.Find(".capa_lista").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Find("a").Attr("href")
+		links = append(links, link)
+	})
+
+	// extract each torrent link
+	indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrentsRedeTorrent(ctx, i, link)
+	})
+
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
+func getTorrentsRedeTorrent(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+	var indexedTorrents []schema.IndexedTorrent
+	doc, err := getDocument(ctx, i, link)
+	if err != nil {
+		return nil, err
+	}
+
+	article := doc.Find(".conteudo")
+	// title pattern: "Something - optional balbla (dddd) some shit" - extract "Something" and "dddd"
+	titleRe := regexp.MustCompile(`^(.*?)(?: - (.*?))? \((\d{4})\)`)
+	titleP := titleRe.FindStringSubmatch(article.Find("h1").Text())
+	if len(titleP) < 3 {
+		return nil, fmt.Errorf("could not extract title from %s", link)
+	}
+	title := strings.TrimSpace(titleP[1])
+	year := strings.TrimSpace(titleP[3])
+
+	textContent := article.Find(".apenas_itemprop")
+	date := getPublishedDateFromMeta(doc)
+	magnets := textContent.Find("a[href^=\"magnet\"]")
+	var magnetLinks []string
+	magnets.Each(func(i int, s *goquery.Selection) {
+		magnetLink, _ := s.Attr("href")
+		magnetLinks = append(magnetLinks, magnetLink)
+	})
+
+	var audio []schema.Audio
+	var size []string
+	article.Find("div#informacoes > p").Each(func(i int, s *goquery.Selection) {
+		// pattern:
+		// Filme Bicho de Sete Cabeças Torrent
+		// Título Original: Bicho de Sete Cabeças
+		// Lançamento: 2001
+		// Gêneros: Drama / Nacional
+		// Idioma: Português
+		// Qualidade: 720p / BluRay
+		// Duração: 1h 14 Minutos
+		// Formato: Mp4
+		// Vídeo: 10 e Áudio: 10
+		// Legendas: Português
+		// Nota do Imdb: 7.7
+		// Tamanho: 1.26 GB
+
+		// we need to manualy parse because the text is not well formatted
+		htmlContent, err := s.Html()
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+
+		// remove any \n and \t characters
+		htmlContent = strings.ReplaceAll(htmlContent, "\n", "")
+		htmlContent = strings.ReplaceAll(htmlContent, "\t", "")
+
+		// split by <br> tags and render each line
+		brRe := regexp.MustCompile(`<br\s*\/?>`)
+		htmlContent = brRe.ReplaceAllString(htmlContent, "<br>")
+		lines := strings.Split(htmlContent, "<br>")
+
+		var text strings.Builder
+		for _, line := range lines {
+			// remove any HTML tags
+			re := regexp.MustCompile(`<[^>]*>`)
+			line = re.ReplaceAllString(line, "")
+
+			line = strings.TrimSpace(line)
+			text.WriteString(line + "\n")
+		}
+
+		audio = append(audio, findAudioFromText(text.String())...)
+		y := findYearFromText(text.String(), title)
+		if y != "" {
+			year = y
+		}
+		size = append(size, findSizesFromText(text.String())...)
+	})
+
+	// find any link from imdb
+	imdbLink := ""
+	article.Find("a").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Attr("href")
+		_imdbLink, err := getIMDBLink(link)
+		if err == nil {
+			imdbLink = _imdbLink
+		}
+	})
+
+	size = utils.StableUniq(size)
+
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
+
+	// for each magnet link, create a new indexed torrent
+	for it, magnetLink := range magnetLinks {
+		it := it
+		go func(it int, magnetLink string) {
+			magnet, err := magnet.ParseMagnetUri(magnetLink)
+			if err != nil {
+				fmt.Println(err)
+			}
+			releaseTitle := magnet.DisplayName
+			infoHash := magnet.InfoHash.String()
+			trackers := magnet.Trackers
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
+
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+			if err != nil {
+				fmt.Println(err)
+			}
+
+			title := processTitle(title, magnetAudio)
+
+			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
+			var mySize string
+			if len(size) == len(magnetLinks) {
+				mySize = size[it]
+			}
+			if mySize == "" {
+				go func() {
+					_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
+				}()
+			}
+
+			ixt := schema.IndexedTorrent{
+				Title:         releaseTitle,
+				OriginalTitle: title,
+				Details:       link,
+				Year:          year,
+				IMDB:          imdbLink,
+				Audio:         magnetAudio,
+				MagnetLink:    magnetLink,
+				Date:          date,
+				InfoHash:      infoHash,
+				Trackers:      trackers,
+				LeechCount:    peer,
+				SeedCount:     seed,
+				Size:          mySize,
+			}
+			chanIndexedTorrent <- ixt
+		}(it, magnetLink)
+	}
+
+	for i := 0; i < len(magnetLinks); i++ {
+		it := <-chanIndexedTorrent
+		indexedTorrents = append(indexedTorrents, it)
+	}
+
+	return indexedTorrents, nil
+}

api/search.go

@@ -0,0 +1,158 @@
+package handler
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+	"time"
+
+	meilisearch "github.com/felipemarinho97/torrent-indexer/search"
+)
+
+// MeilisearchHandler handles HTTP requests for Meilisearch integration.
+type MeilisearchHandler struct {
+	Module *meilisearch.SearchIndexer
+}
+
+// HealthResponse represents the health check response
+type HealthResponse struct {
+	Status    string                 `json:"status"`
+	Service   string                 `json:"service"`
+	Details   map[string]interface{} `json:"details,omitempty"`
+	Timestamp string                 `json:"timestamp"`
+}
+
+// StatsResponse represents the stats endpoint response
+type StatsResponse struct {
+	Status            string           `json:"status"`
+	NumberOfDocuments int64            `json:"numberOfDocuments"`
+	IsIndexing        bool             `json:"isIndexing"`
+	FieldDistribution map[string]int64 `json:"fieldDistribution"`
+	Service           string           `json:"service"`
+}
+
+// NewMeilisearchHandler creates a new instance of MeilisearchHandler.
+func NewMeilisearchHandler(module *meilisearch.SearchIndexer) *MeilisearchHandler {
+	return &MeilisearchHandler{Module: module}
+}
+
+// SearchTorrentHandler handles the searching of torrent items.
+func (h *MeilisearchHandler) SearchTorrentHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	query := r.URL.Query().Get("q")
+	if query == "" {
+		http.Error(w, "Query parameter 'q' is required", http.StatusBadRequest)
+		return
+	}
+
+	limitStr := r.URL.Query().Get("limit")
+	limit := 10 // Default limit
+	if limitStr != "" {
+		var err error
+		limit, err = strconv.Atoi(limitStr)
+		if err != nil || limit <= 0 {
+			http.Error(w, "Invalid limit parameter", http.StatusBadRequest)
+			return
+		}
+	}
+
+	results, err := h.Module.SearchTorrent(query, limit)
+	if err != nil {
+		http.Error(w, "Failed to search torrents", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(results); err != nil {
+		http.Error(w, "Failed to encode response", http.StatusInternalServerError)
+	}
+}
+
+// HealthHandler provides a health check endpoint for Meilisearch.
+func (h *MeilisearchHandler) HealthHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	// Check if Meilisearch is healthy
+	isHealthy := h.Module.IsHealthy()
+
+	response := HealthResponse{
+		Service:   "meilisearch",
+		Timestamp: getCurrentTimestamp(),
+	}
+
+	if isHealthy {
+		// Try to get additional stats for more detailed health info
+		stats, err := h.Module.GetStats()
+		if err == nil {
+			response.Status = "healthy"
+			response.Details = map[string]interface{}{
+				"documents": stats.NumberOfDocuments,
+				"indexing":  stats.IsIndexing,
+			}
+			w.WriteHeader(http.StatusOK)
+		} else {
+			// Service is up but can't get stats
+			response.Status = "degraded"
+			response.Details = map[string]interface{}{
+				"error": "Could not retrieve stats",
+			}
+			w.WriteHeader(http.StatusOK)
+		}
+	} else {
+		// Service is down
+		response.Status = "unhealthy"
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}
+
+	if err := json.NewEncoder(w).Encode(response); err != nil {
+		http.Error(w, "Failed to encode response", http.StatusInternalServerError)
+	}
+}
+
+// StatsHandler provides detailed statistics about the Meilisearch index.
+func (h *MeilisearchHandler) StatsHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	// Get detailed stats from Meilisearch
+	stats, err := h.Module.GetStats()
+	if err != nil {
+		// Check if it's a connectivity issue
+		if !h.Module.IsHealthy() {
+			http.Error(w, "Meilisearch service is unavailable", http.StatusServiceUnavailable)
+			return
+		}
+		http.Error(w, "Failed to retrieve statistics", http.StatusInternalServerError)
+		return
+	}
+
+	response := StatsResponse{
+		Status:            "healthy",
+		Service:           "meilisearch",
+		NumberOfDocuments: stats.NumberOfDocuments,
+		IsIndexing:        stats.IsIndexing,
+		FieldDistribution: stats.FieldDistribution,
+	}
+
+	if err := json.NewEncoder(w).Encode(response); err != nil {
+		http.Error(w, "Failed to encode response", http.StatusInternalServerError)
+	}
+}
+
+// getCurrentTimestamp returns the current timestamp in RFC3339 format
+func getCurrentTimestamp() string {
+	return time.Now().Format(time.RFC3339)
+}

api/starck_filmes.go

@@ -0,0 +1,226 @@
+package handler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+var starck_filmes = IndexerMeta{
+	Label:       "starck_filmes",
+	URL:         "https://www.starckfilmes.online/",
+	SearchURL:   "?s=",
+	PagePattern: "page/%s",
+}
+
+func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	metadata := starck_filmes
+
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
+	}()
+
+	ctx := r.Context()
+	// supported query params: q, page, filter_results
+	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
+
+	// URL encode query param
+	q = url.QueryEscape(q)
+	url := metadata.URL
+	if q != "" {
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
+	}
+
+	fmt.Println("URL:>", url)
+	resp, err := i.requester.GetDocument(ctx, url)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
+		return
+	}
+	defer resp.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
+		return
+	}
+
+	var links []string
+	doc.Find(".item").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Find("div.sub-item > a").Attr("href")
+		links = append(links, link)
+	})
+
+	// extract each torrent link
+	indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrentStarckFilmes(ctx, i, link)
+	})
+
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
+func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+	var indexedTorrents []schema.IndexedTorrent
+	doc, err := getDocument(ctx, i, link)
+	if err != nil {
+		return nil, err
+	}
+
+	date := getPublishedDateFromRawString(link)
+
+	post := doc.Find(".post")
+	capa := post.Find(".capa")
+	title := capa.Find(".post-description > h2").Text()
+	post_buttons := post.Find(".post-buttons")
+	magnets := post_buttons.Find("a[href^=\"magnet\"]")
+	var magnetLinks []string
+	magnets.Each(func(i int, s *goquery.Selection) {
+		magnetLink, _ := s.Attr("href")
+		magnetLinks = append(magnetLinks, magnetLink)
+	})
+
+	var audio []schema.Audio
+	var year string
+	var size []string
+	capa.Find(".post-description p").Each(func(i int, s *goquery.Selection) {
+		// pattern:
+		// Nome Original: 28 Weeks Later
+		// Lançamento: 2007
+		// Duração: 1h 40 min
+		// Gênero: Terror, Suspense, Ficção
+		// Formato: MKV
+		// Tamanho: 2.45 GB
+		// Qualidade de Video: 10
+		// Qualidade do Audio: 10
+		// Idioma: Português | Inglês
+		// Legenda: Português, Inglês, Espanhol
+		var text strings.Builder
+		s.Find("span").Each(func(i int, span *goquery.Selection) {
+			text.WriteString(span.Text())
+			text.WriteString(" ")
+		})
+		audio = append(audio, findAudioFromText(text.String())...)
+		y := findYearFromText(text.String(), title)
+		if y != "" {
+			year = y
+		}
+		size = append(size, findSizesFromText(text.String())...)
+	})
+
+	// TODO: find any link from imdb
+	imdbLink := ""
+
+	size = utils.StableUniq(size)
+
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
+
+	// for each magnet link, create a new indexed torrent
+	for it, magnetLink := range magnetLinks {
+		it := it
+		go func(it int, magnetLink string) {
+			magnet, err := magnet.ParseMagnetUri(magnetLink)
+			if err != nil {
+				fmt.Println(err)
+			}
+			releaseTitle := strings.TrimSpace(magnet.DisplayName)
+			// url decode the title
+			releaseTitle, err = url.QueryUnescape(releaseTitle)
+			if err != nil {
+				fmt.Println(err)
+				releaseTitle = strings.TrimSpace(magnet.DisplayName)
+			}
+			infoHash := magnet.InfoHash.String()
+			trackers := magnet.Trackers
+			for i, tracker := range trackers {
+				unescapedTracker, err := url.QueryUnescape(tracker)
+				if err != nil {
+					fmt.Println(err)
+				}
+				trackers[i] = strings.TrimSpace(unescapedTracker)
+			}
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
+
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+			if err != nil {
+				fmt.Println(err)
+			}
+
+			title := processTitle(title, magnetAudio)
+
+			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
+			var mySize string
+			if len(size) == len(magnetLinks) {
+				mySize = size[it]
+			}
+			if mySize == "" {
+				go func() {
+					_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
+				}()
+			}
+
+			ixt := schema.IndexedTorrent{
+				Title:         releaseTitle,
+				OriginalTitle: title,
+				Details:       link,
+				Year:          year,
+				IMDB:          imdbLink,
+				Audio:         magnetAudio,
+				MagnetLink:    magnetLink,
+				Date:          date,
+				InfoHash:      infoHash,
+				Trackers:      trackers,
+				LeechCount:    peer,
+				SeedCount:     seed,
+				Size:          mySize,
+			}
+			chanIndexedTorrent <- ixt
+		}(it, magnetLink)
+	}
+
+	for i := 0; i < len(magnetLinks); i++ {
+		it := <-chanIndexedTorrent
+		indexedTorrents = append(indexedTorrents, it)
+	}
+
+	return indexedTorrents, nil
+}

api/torrent_dos_filmes.go

@@ -0,0 +1,220 @@
+package handler
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/schema"
+	goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+var torrent_dos_filmes = IndexerMeta{
+	Label:       "torrent_dos_filmes",
+	URL:         "https://torrentdosfilmes.se/",
+	SearchURL:   "?s=",
+	PagePattern: "category/dublado/page/%s",
+}
+
+func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.Request) {
+	start := time.Now()
+	metadata := torrent_dos_filmes
+
+	defer func() {
+		i.metrics.IndexerDuration.WithLabelValues(metadata.Label).Observe(time.Since(start).Seconds())
+		i.metrics.IndexerRequests.WithLabelValues(metadata.Label).Inc()
+	}()
+
+	ctx := r.Context()
+	// supported query params: q, season, episode, page, filter_results
+	q := r.URL.Query().Get("q")
+	page := r.URL.Query().Get("page")
+
+	// URL encode query param
+	q = url.QueryEscape(q)
+	url := metadata.URL
+	if q != "" {
+		url = fmt.Sprintf("%s%s%s", url, metadata.SearchURL, q)
+	} else if page != "" {
+		url = fmt.Sprintf(fmt.Sprintf("%s%s", url, metadata.PagePattern), page)
+	}
+
+	fmt.Println("URL:>", url)
+	resp, err := i.requester.GetDocument(ctx, url)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
+		return
+	}
+	defer resp.Close()
+
+	doc, err := goquery.NewDocumentFromReader(resp)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		i.metrics.IndexerErrors.WithLabelValues(metadata.Label).Inc()
+		return
+	}
+
+	var links []string
+	doc.Find(".post").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Find("div.title > a").Attr("href")
+		links = append(links, link)
+	})
+
+	// extract each torrent link
+	indexedTorrents := utils.ParallelFlatMap(links, func(link string) ([]schema.IndexedTorrent, error) {
+		return getTorrentsTorrentDosFilmes(ctx, i, link)
+	})
+
+	// Apply post-processors
+	postProcessedTorrents := indexedTorrents
+	for _, processor := range i.postProcessors {
+		postProcessedTorrents = processor(i, r, postProcessedTorrents)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	err = json.NewEncoder(w).Encode(Response{
+		Results: postProcessedTorrents,
+		Count:   len(postProcessedTorrents),
+	})
+	if err != nil {
+		fmt.Println(err)
+	}
+}
+
+func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
+	var indexedTorrents []schema.IndexedTorrent
+	doc, err := getDocument(ctx, i, link)
+	if err != nil {
+		return nil, err
+	}
+
+	article := doc.Find("article")
+	title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1)
+	textContent := article.Find("div.content")
+	date := getPublishedDateFromMeta(doc)
+	magnets := textContent.Find("a[href^=\"magnet\"]")
+	var magnetLinks []string
+	magnets.Each(func(i int, s *goquery.Selection) {
+		magnetLink, _ := s.Attr("href")
+		magnetLinks = append(magnetLinks, magnetLink)
+	})
+
+	var audio []schema.Audio
+	var year string
+	var size []string
+	article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
+		// pattern:
+		// Título Traduzido: Fundação
+		// Título Original: Foundation
+		// IMDb: 7,5
+		// Ano de Lançamento: 2023
+		// Gênero: Ação | Aventura | Ficção
+		// Formato: MKV
+		// Qualidade: WEB-DL
+		// Áudio: Português | Inglês
+		// Idioma: Português | Inglês
+		// Legenda: Português
+		// Tamanho: –
+		// Qualidade de Áudio: 10
+		// Qualidade de Vídeo: 10
+		// Duração: 59 Min.
+		// Servidor: Torrent
+		text := s.Text()
+
+		audio = append(audio, findAudioFromText(text)...)
+		y := findYearFromText(text, title)
+		if y != "" {
+			year = y
+		}
+		size = append(size, findSizesFromText(text)...)
+	})
+
+	// find any link from imdb
+	imdbLink := ""
+	article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
+		link, _ := s.Attr("href")
+		_imdbLink, err := getIMDBLink(link)
+		if err == nil {
+			imdbLink = _imdbLink
+		}
+	})
+
+	size = utils.StableUniq(size)
+
+	var chanIndexedTorrent = make(chan schema.IndexedTorrent)
+
+	// for each magnet link, create a new indexed torrent
+	for it, magnetLink := range magnetLinks {
+		it := it
+		go func(it int, magnetLink string) {
+			magnet, err := magnet.ParseMagnetUri(magnetLink)
+			if err != nil {
+				fmt.Println(err)
+			}
+			releaseTitle := magnet.DisplayName
+			infoHash := magnet.InfoHash.String()
+			trackers := magnet.Trackers
+			magnetAudio := getAudioFromTitle(releaseTitle, audio)
+
+			peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
+			if err != nil {
+				fmt.Println(err)
+			}
+
+			title := processTitle(title, magnetAudio)
+
+			// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
+			var mySize string
+			if len(size) == len(magnetLinks) {
+				mySize = size[it]
+			}
+			if mySize == "" {
+				go func() {
+					_, _ = i.magnetMetadataAPI.FetchMetadata(ctx, magnetLink)
+				}()
+			}
+
+			ixt := schema.IndexedTorrent{
+				Title:         releaseTitle,
+				OriginalTitle: title,
+				Details:       link,
+				Year:          year,
+				IMDB:          imdbLink,
+				Audio:         magnetAudio,
+				MagnetLink:    magnetLink,
+				Date:          date,
+				InfoHash:      infoHash,
+				Trackers:      trackers,
+				LeechCount:    peer,
+				SeedCount:     seed,
+				Size:          mySize,
+			}
+			chanIndexedTorrent <- ixt
+		}(it, magnetLink)
+	}
+
+	for i := 0; i < len(magnetLinks); i++ {
+		it := <-chanIndexedTorrent
+		indexedTorrents = append(indexedTorrents, it)
+	}
+
+	return indexedTorrents, nil
+}

cache/redis.go

@@ -9,13 +9,14 @@ import (
 	"github.com/redis/go-redis/v9"
 )
 
-var (
-	DefaultExpiration      = 24 * time.Hour * 180 // 180 days
+const (
+	DefaultExpiration      = 24 * time.Hour * 7 // 7 days
 	IndexerComandoTorrents = "indexer:comando_torrents"
 )
 
 type Redis struct {
-	client *redis.Client
+	client            *redis.Client
+	defaultExpiration time.Duration
 }
 
 func NewRedis() *Redis {
@@ -28,15 +29,20 @@ func NewRedis() *Redis {
 			Addr:     fmt.Sprintf("%s:6379", redisHost),
 			Password: "",
 		}),
+		defaultExpiration: DefaultExpiration,
 	}
 }
 
+func (r *Redis) SetDefaultExpiration(expiration time.Duration) {
+	r.defaultExpiration = expiration
+}
+
 func (r *Redis) Get(ctx context.Context, key string) ([]byte, error) {
 	return r.client.Get(ctx, key).Bytes()
 }
 
 func (r *Redis) Set(ctx context.Context, key string, value []byte) error {
-	return r.client.Set(ctx, key, value, DefaultExpiration).Err()
+	return r.client.Set(ctx, key, value, r.defaultExpiration).Err()
 }
 
 func (r *Redis) SetWithExpiration(ctx context.Context, key string, value []byte, expiration time.Duration) error {

consts/version.go

@@ -0,0 +1,14 @@
+package consts
+
+// These will be injected via -ldflags at build time
+var (
+	gitSha string = "unknown"
+	gitTag string = "unknown"
+)
+
+func GetBuildInfo() map[string]string {
+	return map[string]string{
+		"revision": gitSha,
+		"version":  gitTag,
+	}
+}

docker-compose.yml

@@ -1,11 +1,8 @@
-version: '3.7'
+version: '3'
 
 services:
   torrent-indexer:
-    image:
-      build:
-        context: .
-        dockerfile: Dockerfile
+    image: felipemarinho97/torrent-indexer:latest
     container_name: torrent-indexer
     restart: unless-stopped
     ports:
@@ -14,6 +11,16 @@ services:
       - indexer
     environment:
       - REDIS_HOST=redis
+      - FLARESOLVERR_ADDRESS=http://flaresolverr:8191
+
+      ## Meilisearch configuration (optional)
+      # - MEILISEARCH_ADDRESS=http://meilisearch:7700
+      # - MEILISEARCH_KEY=my-secret-key
+
+      ## Magnet Metadata API configuration (optional)
+      # - MAGNET_METADATA_API_ENABLED=false
+      # - MAGNET_METADATA_API_ADDRESS=http://magnet-metadata-api:8080
+      # - MAGNET_METADATA_API_TIMEOUT_SECONDS=10
   
   redis:
     image: redis:alpine
@@ -22,5 +29,45 @@ services:
     networks:
       - indexer
 
+  ##### MEILISEARCH #####
+  ## This container is not necessary for the indexer to work,
+  ## deploy if you want to use the search feature
+  #
+  # meilisearch: 
+  #   image: getmeili/meilisearch:latest
+  #   container_name: meilisearch
+  #   restart: unless-stopped
+  #   networks:
+  #     - indexer
+  #   environment:
+  #     - MEILI_NO_ANALYTICS=true
+  #     - MEILI_MASTER_KEY=my-secret-key
+
+  ##### MAGNET METADATA API #####
+  ## This container is not necessary for the indexer to work,
+  ## deploy if you want to fetch metadata from p2p network
+  ## CAUTION: Never deploy this container on a cloud server (AWS, GCP, Azure, Oracle), or you will get banned!
+  #
+  # magnet-metadata-api:
+  #   image: felipemarinho97/magnet-metadata-api:latest
+  #   container_name: magnet-metadata-api
+  #   restart: unless-stopped
+  #   ports:
+  #     - "8999:8080"
+  #     - "42069:42069"
+  #   networks:
+  #     - indexer
+  #   environment:
+  #     - PORT=8080
+  #     - REDIS_URL=redis://redis:6379
+  #     - CACHE_DIR=/home/torrent/cache
+  #     - ENABLE_DOWNLOADS=false
+  #     - DOWNLOAD_BASE_URL=http://localhost:8999
+  #     - CLIENT_PORT=42069
+  #     - SEEDING_ENABLED=false
+  #     - FALLBACK_INITIAL_CHUNK_SIZE_KB=24
+  #   volumes:
+  #     - ./magnet-metadata-cache:/home/torrent/cache
+
 networks:
   indexer:

go.mod

@@ -1,14 +1,26 @@
 module github.com/felipemarinho97/torrent-indexer
 
-go 1.18
+go 1.24
 
-require github.com/redis/go-redis/v9 v9.2.0
+require github.com/redis/go-redis/v9 v9.11.0
 
 require (
-	github.com/andybalholm/cascadia v1.3.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	golang.org/x/net v0.7.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.65.0 // indirect
+	github.com/prometheus/procfs v0.17.0 // indirect
+	golang.org/x/sys v0.34.0 // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
 )
 
-require github.com/PuerkitoBio/goquery v1.8.1
+require (
+	github.com/PuerkitoBio/goquery v1.10.3
+	github.com/hbollon/go-edlib v1.6.0
+	github.com/prometheus/client_golang v1.22.0
+	github.com/xhit/go-str2duration/v2 v2.1.0
+	golang.org/x/net v0.42.0
+)

go.sum

@@ -1,45 +1,115 @@
-github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
-github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
-github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
-github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
+github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/redis/go-redis/v9 v9.2.0 h1:zwMdX0A4eVzse46YN18QhuDiM4uf3JmkOB4VZrdt5uI=
-github.com/redis/go-redis/v9 v9.2.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/hbollon/go-edlib v1.6.0 h1:ga7AwwVIvP8mHm9GsPueC0d71cfRU/52hmPJ7Tprv4E=
+github.com/hbollon/go-edlib v1.6.0/go.mod h1:wnt6o6EIVEzUfgbUZY7BerzQ2uvzp354qmS2xaLkrhM=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
+github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
+github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
+github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
+github.com/redis/go-redis/v9 v9.11.0 h1:E3S08Gl/nJNn5vkxd2i78wZxWAPNZgUNTp8WIJUAiIs=
+github.com/redis/go-redis/v9 v9.11.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
+github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
+golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
+golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
+google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

magnet/magnet_metadata_api.go

@@ -0,0 +1,113 @@
+package magnet
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/felipemarinho97/torrent-indexer/cache"
+)
+
+type MetadataRequest struct {
+	MagnetURI string `json:"magnet_uri"`
+}
+
+type TorrentFile struct {
+	Path   string `json:"path"`
+	Size   int64  `json:"size"`
+	Offset int64  `json:"offset"`
+}
+
+type MetadataResponse struct {
+	InfoHash    string        `json:"info_hash"`
+	Name        string        `json:"name"`
+	Size        int64         `json:"size"`
+	Files       []TorrentFile `json:"files"`
+	CreatedBy   string        `json:"created_by"`
+	CreatedAt   time.Time     `json:"created_at"`
+	Comment     string        `json:"comment"`
+	Trackers    []string      `json:"trackers"`
+	DownloadURL string        `json:"download_url"`
+}
+
+type MetadataClient struct {
+	baseURL    string
+	httpClient *http.Client
+	c          *cache.Redis
+}
+
+func NewClient(baseURL string, timeout time.Duration, c *cache.Redis) *MetadataClient {
+	return &MetadataClient{
+		baseURL: baseURL,
+		httpClient: &http.Client{
+			Timeout: timeout,
+			Transport: &http.Transport{
+				MaxIdleConns:      100,
+				IdleConnTimeout:   30 * time.Second,
+				ForceAttemptHTTP2: true,
+			},
+		},
+		c: c,
+	}
+}
+
+func (c *MetadataClient) IsEnabled() bool {
+	return c != nil && c.baseURL != ""
+}
+
+func (c *MetadataClient) FetchMetadata(ctx context.Context, magnetURI string) (*MetadataResponse, error) {
+	if !c.IsEnabled() {
+		return nil, fmt.Errorf("magnet metadata API is not enabled")
+	}
+	// Check cache first
+	m, err := ParseMagnetUri(magnetURI)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse magnet URI: %w", err)
+	}
+	cacheKey := fmt.Sprintf("metadata:%s", m.InfoHash)
+	cachedData, err := c.c.Get(ctx, cacheKey)
+	if err == nil && cachedData != nil {
+		var cachedMetadata MetadataResponse
+		if err := json.Unmarshal(cachedData, &cachedMetadata); err == nil {
+			return &cachedMetadata, nil
+		}
+	}
+
+	reqBody := MetadataRequest{MagnetURI: magnetURI}
+	body, err := json.Marshal(reqBody)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal request body: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/api/v1/metadata", bytes.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send POST request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("API responded with status: %s", resp.Status)
+	}
+
+	var metadata MetadataResponse
+	if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
+		return nil, fmt.Errorf("failed to decode response: %w", err)
+	}
+
+	// Cache the metadata response
+	cacheData, err := json.Marshal(metadata)
+	if err == nil {
+		_ = c.c.SetWithExpiration(ctx, cacheKey, cacheData, 7*24*time.Hour)
+	}
+
+	return &metadata, nil
+}

main.go

@@ -1,21 +1,92 @@
 package main
 
 import (
+	"fmt"
 	"net/http"
+	"os"
+	"strconv"
+	"time"
 
 	handler "github.com/felipemarinho97/torrent-indexer/api"
 	"github.com/felipemarinho97/torrent-indexer/cache"
+	"github.com/felipemarinho97/torrent-indexer/magnet"
+	"github.com/felipemarinho97/torrent-indexer/monitoring"
+	"github.com/felipemarinho97/torrent-indexer/public"
+	"github.com/felipemarinho97/torrent-indexer/requester"
+	meilisearch "github.com/felipemarinho97/torrent-indexer/search"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	str2duration "github.com/xhit/go-str2duration/v2"
 )
 
 func main() {
 	redis := cache.NewRedis()
-	indexers := handler.NewIndexers(redis)
+	searchIndex := meilisearch.NewSearchIndexer(os.Getenv("MEILISEARCH_ADDRESS"), os.Getenv("MEILISEARCH_KEY"), "torrents")
+	var magnetMetadataAPI *magnet.MetadataClient
+	if os.Getenv("MAGNET_METADATA_API_ENABLED") == "true" {
+		timeout := 10 * time.Second
+		if v := os.Getenv("MAGNET_METADATA_API_TIMEOUT_SECONDS"); v != "" {
+			if t, err := strconv.Atoi(v); err == nil {
+				timeout = time.Duration(t) * time.Second
+			}
+		}
+		magnetMetadataAPI = magnet.NewClient(os.Getenv("MAGNET_METADATA_API_ADDRESS"), timeout, redis)
+	}
+	metrics := monitoring.NewMetrics()
+	metrics.Register()
 
-	http.HandleFunc("/", handler.HandlerIndex)
-	http.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
-	http.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
+	flaresolverr := requester.NewFlareSolverr(os.Getenv("FLARESOLVERR_ADDRESS"), 60000)
+	req := requester.NewRequester(flaresolverr, redis)
 
-	err := http.ListenAndServe(":7006", nil)
+	// get shot-lived and long-lived cache expiration from env
+	shortLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("SHORT_LIVED_CACHE_EXPIRATION"))
+	if err == nil {
+		fmt.Printf("Setting short-lived cache expiration to %s\n", shortLivedCacheExpiration)
+		req.SetShortLivedCacheExpiration(shortLivedCacheExpiration)
+	}
+	longLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("LONG_LIVED_CACHE_EXPIRATION"))
+	if err == nil {
+		fmt.Printf("Setting long-lived cache expiration to %s\n", longLivedCacheExpiration)
+		redis.SetDefaultExpiration(longLivedCacheExpiration)
+	} else {
+		fmt.Println(err)
+	}
+
+	indexers := handler.NewIndexers(redis, metrics, req, searchIndex, magnetMetadataAPI)
+	search := handler.NewMeilisearchHandler(searchIndex)
+
+	indexerMux := http.NewServeMux()
+	metricsMux := http.NewServeMux()
+
+	indexerMux.HandleFunc("/", handler.HandlerIndex)
+	indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
+	indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
+	indexerMux.HandleFunc("/indexers/comandohds", indexers.HandlerComandoHDsIndexer)
+	indexerMux.HandleFunc("/indexers/rede_torrent", indexers.HandlerRedeTorrentIndexer)
+	indexerMux.HandleFunc("/indexers/starck-filmes", indexers.HandlerStarckFilmesIndexer)
+	indexerMux.HandleFunc("/indexers/torrent-dos-filmes", indexers.HandlerTorrentDosFilmesIndexer)
+	indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer)
+	indexerMux.HandleFunc("/search", search.SearchTorrentHandler)
+	indexerMux.HandleFunc("/search/health", search.HealthHandler)
+	indexerMux.HandleFunc("/search/stats", search.StatsHandler)
+	indexerMux.Handle("/ui/", http.StripPrefix("/ui/", http.FileServer(http.FS(public.UIFiles))))
+
+	metricsMux.Handle("/metrics", promhttp.Handler())
+
+	go func() {
+		err := http.ListenAndServe(":8081", metricsMux)
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	port := os.Getenv("PORT")
+	if port == "" {
+		port = "7006"
+	}
+
+	fmt.Printf("Server listening on :%s\n", port)
+	err = http.ListenAndServe(":"+port, indexerMux)
 	if err != nil {
 		panic(err)
 	}

monitoring/prometheus.go

@@ -0,0 +1,47 @@
+package monitoring
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type Metrics struct {
+	IndexerDuration *prometheus.HistogramVec
+	IndexerErrors   *prometheus.CounterVec
+	IndexerRequests *prometheus.CounterVec
+	CacheHits       *prometheus.CounterVec
+	CacheMisses     *prometheus.CounterVec
+}
+
+func NewMetrics() *Metrics {
+	return &Metrics{
+		IndexerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "indexer_duration_seconds",
+			Help:    "Duration of indexer requests",
+			Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 20, 30},
+		}, []string{"indexer"}),
+		IndexerErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "indexer_errors_total",
+			Help: "Number of indexer errors",
+		}, []string{"indexer"}),
+		IndexerRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "indexer_requests_total",
+			Help: "Number of indexer requests",
+		}, []string{"indexer"}),
+		CacheHits: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "cache_hits_total",
+			Help: "Number of cache hits",
+		}, []string{"cache"}),
+		CacheMisses: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "cache_misses_total",
+			Help: "Number of cache misses",
+		}, []string{"cache"}),
+	}
+}
+
+func (m *Metrics) Register() {
+	prometheus.MustRegister(m.IndexerDuration)
+	prometheus.MustRegister(m.IndexerErrors)
+	prometheus.MustRegister(m.IndexerRequests)
+	prometheus.MustRegister(m.CacheHits)
+	prometheus.MustRegister(m.CacheMisses)
+}

public/index.go

@@ -0,0 +1,6 @@
+package public
+
+import "embed"
+
+//go:embed *
+var UIFiles embed.FS

public/index.html

@@ -0,0 +1,226 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Torrent Indexer</title>
+  <script src="https://cdn.tailwindcss.com"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@heroicons/react/solid@2.0.0/dist/index.umd.js"></script>
+</head>
+
+<body class="bg-gray-900 text-white font-sans min-h-screen flex flex-col">
+  <div class="container mx-auto p-6 flex-grow">
+    <!-- Header -->
+    <header class="text-center mb-10">
+      <h1 class="text-4xl font-bold text-blue-400">Torrent Indexer 🇧🇷</h1>
+      <p class="text-gray-400 mt-2">Find torrents with detailed information from torrent-indexer cache</p>
+    </header>
+
+    <!-- Search Bar -->
+    <div class="flex justify-center mb-10">
+      <input id="search-query" type="text" placeholder="Enter search query"
+        class="w-full max-w-lg px-4 py-2 rounded-md border border-gray-600 bg-gray-800 text-white focus:ring focus:ring-blue-500">
+      <button id="search-btn"
+        class="ml-4 px-6 py-2 bg-blue-600 hover:bg-blue-700 rounded-md font-bold text-white">Search</button>
+    </div>
+    
+    <!-- Results Section -->
+    <div id="results" class="space-y-6 mb-10">
+      <!-- Dynamic content will be injected here -->
+    </div>
+
+    <!-- Health Warning -->
+    <div id="health-warning" class="hidden mb-6 p-4 bg-yellow-800 border border-yellow-600 rounded-lg">
+      <div class="flex items-center gap-2">
+        <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5 text-yellow-400" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+          <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 9v2m0 4h.01m-9 4h18a2 2 0 002-2V7a2 2 0 00-2-2H3a2 2 0 00-2 2v10a2 2 0 002 2z" />
+        </svg>
+        <span class="font-bold text-yellow-400">Service Warning</span>
+      </div>
+      <p class="text-yellow-200 mt-2">Search functionality may be disabled or experiencing issues. Please try again later.</p>
+    </div>
+
+  </div>
+  <!-- Stats Section -->
+  <div id="database-statistics" class="stats-info mt-auto mb-1 p-3 rounded text-center">
+    <span id="torrentStats" class="text-gray-400">Loading stats...</span>
+  </div>
+
+  <script>
+    // Global variables
+    let serviceHealthy = true;
+
+    // Function to check service health
+    async function checkHealth() {
+      try {
+        const response = await fetch('/search/health');
+        const health = await response.json();
+        
+        if (response.status === 503 || health.status === 'unhealthy') {
+          serviceHealthy = false;
+          showHealthWarning();
+          hideDatabaseStatistics();
+        } else if (health.status === 'degraded') {
+          serviceHealthy = true; // Still operational
+          showHealthWarning(); // But show warning
+        } else {
+          serviceHealthy = true;
+          hideHealthWarning();
+        }
+      } catch (error) {
+        serviceHealthy = false;
+        showHealthWarning();
+        console.error('Health check failed:', error);
+      }
+    }
+
+    // Function to show health warning
+    function showHealthWarning() {
+      document.getElementById('health-warning').classList.remove('hidden');
+    }
+
+    // Function to hide health warning
+    function hideHealthWarning() {
+      document.getElementById('health-warning').classList.add('hidden');
+    }
+
+    function hideDatabaseStatistics() {
+      document.getElementById('database-statistics').classList.add('hidden');
+    }
+
+    // Function to load stats
+    async function loadStats() {
+      try {
+        const response = await fetch('/search/stats');
+        if (response.ok) {
+          const stats = await response.json();
+          const statsElement = document.getElementById('torrentStats');
+          
+          const formattedStats = `
+              <span class="text-sm text-gray-500">
+                <span class="text-green-400 font-medium">${stats.numberOfDocuments?.toLocaleString()+'+' || 'N/A'}</span> indexed torrents!
+              </span>
+          `;
+          
+          statsElement.innerHTML = formattedStats;
+        } else {
+          throw new Error('Failed to load stats');
+        }
+      } catch (error) {
+        hideDatabaseStatistics();
+        console.error('Stats loading failed:', error);
+      }
+    }
+
+    // Function to render a single torrent result
+    function renderTorrent(torrent) {
+      return `
+        <div class="p-6 bg-gray-800 rounded-lg shadow-md flex flex-col md:flex-row gap-6">
+          <!-- Torrent Title and Details -->
+          <div class="flex-grow">
+            <h2 class="text-2xl font-bold text-blue-400 flex items-center gap-2">
+              <span>${torrent.title}</span>
+              <span class="text-sm text-gray-400">(${torrent.year})</span>
+            </h2>
+            <p class="text-gray-500 italic mt-1">${torrent.original_title}</p>
+            <div class="mt-4 grid grid-cols-1 sm:grid-cols-2 gap-4">
+              <p><strong>Audio:</strong> ${torrent.audio.join(', ')}</p>
+              <p><strong>Size:</strong> ${torrent.size}</p>
+              <p><strong>Seeds:</strong> ${torrent.seed_count} | <strong>Leeches:</strong> ${torrent.leech_count}</p>
+              <p><strong>Info Hash:</strong> <span class="text-sm break-all text-gray-300">${torrent.info_hash}</span></p>
+            </div>
+          </div>
+
+          <!-- Actions -->
+          <div class="flex flex-col justify-between items-start md:items-end">
+            <div>
+              <a href="${torrent.imdb}" target="_blank"
+                class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium">
+                <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                  <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                    d="M8 16l-4-4m0 0l4-4m-4 4h16" />
+                </svg>
+                View on IMDB
+              </a>
+              <a href="${torrent.details}" target="_blank"
+                class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium mt-2">
+                <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                  <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                    d="M13 16h-1v-4h-.01M9 20h6a2 2 0 002-2v-5a2 2 0 00-2-2h-3.5a2 2 0 00-1.85 1.19M13 10V6a3 3 0 00-6 0v4" />
+                </svg>
+                View Details
+              </a>
+            </div>
+            <a href="${torrent.magnet_link}" target="_blank"
+              class="px-4 py-2 bg-green-600 hover:bg-green-700 text-white font-bold rounded-md flex items-center gap-2 mt-4">
+              <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+                  d="M9 17v-6m6 6v-6m-6 6l-2-2m8 0l2-2M5 9l7-7 7 7" />
+              </svg>
+              Download Magnet
+            </a>
+          </div>
+        </div>
+      `;
+    }
+
+    // Handle search
+    async function onSearch() {
+      const query = document.getElementById('search-query').value.trim();
+      if (!query) {
+        alert('Please enter a search query!');
+        return;
+      }
+
+      try {
+        const response = await fetch(`/search?q=${encodeURIComponent(query)}`);
+        if (!response.ok) {
+          throw new Error('Search failed');
+        }
+
+        const results = await response.json();
+        const resultsContainer = document.getElementById('results');
+        
+        if (results.length === 0) {
+          resultsContainer.innerHTML = `
+            <div class="p-6 bg-gray-800 rounded-lg shadow-md text-center">
+              <p class="text-xl font-bold text-gray-400">No results found</p>
+              <p class="text-gray-500 mt-2">Try different search terms or check spelling</p>
+            </div>
+          `;
+        } else {
+          resultsContainer.innerHTML = results.map(renderTorrent).join('');
+        }
+      } catch (error) {
+        document.getElementById('results').innerHTML = `
+          <div class="p-6 bg-red-800 rounded-lg shadow-md text-center">
+            <p class="text-xl font-bold text-red-400">Error fetching search results</p>
+            <p class="text-gray-400 mt-2">Please try again later.</p>
+          </div>
+        `;
+        console.error(error);
+      }
+    }
+    
+    // Event listeners
+    document.getElementById('search-btn').addEventListener('click', onSearch);
+    document.getElementById('search-query').addEventListener('keypress', (e) => {
+      if (e.key === 'Enter') {
+        onSearch();
+      }
+    });
+
+    // Initialize page
+    document.addEventListener('DOMContentLoaded', async () => {
+      await checkHealth();
+      await loadStats();
+      
+      // Refresh health and stats periodically
+      setInterval(checkHealth, 30000); // Check health every 30 seconds
+      setInterval(loadStats, 60000);   // Update stats every minute
+    });
+  </script>
+</body>
+
+</html>

requester/flaresolverr.go

@@ -0,0 +1,300 @@
+package requester
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"strings"
+	"sync"
+
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+type FlareSolverr struct {
+	url         string
+	maxTimeout  int
+	httpClient  *http.Client
+	sessionPool chan string
+	mu          sync.Mutex
+	initiated   bool
+}
+
+var (
+	ErrListSessions = fmt.Errorf("failed to list sessions")
+)
+
+func NewFlareSolverr(url string, timeoutMilli int) *FlareSolverr {
+	poolSize := 5
+	httpClient := &http.Client{}
+	sessionPool := make(chan string, poolSize) // Pool size of 5 sessions
+
+	f := &FlareSolverr{
+		url:         url,
+		maxTimeout:  timeoutMilli,
+		httpClient:  httpClient,
+		sessionPool: sessionPool,
+	}
+
+	err := f.FillSessionPool()
+	if err == nil {
+		f.initiated = true
+	}
+
+	return f
+}
+
+func (f *FlareSolverr) FillSessionPool() error {
+	// Check if the pool is already filled
+	if len(f.sessionPool) == cap(f.sessionPool) {
+		return nil
+	}
+
+	// Pre-initialize the pool with existing sessions
+	sessions, err := f.ListSessions()
+	if err != nil {
+		// if fail to list sessions, it may not support the sessions.list command
+		// create new dumb sessions to fill the pool
+		if err == ErrListSessions {
+			for len(f.sessionPool) < cap(f.sessionPool) {
+				f.sessionPool <- "dumb-session"
+			}
+			return nil
+		}
+		fmt.Println("Failed to list existing FlareSolverr sessions:", err)
+		return err
+	} else {
+		for _, session := range sessions {
+			// Add available sessions to the pool
+			if len(f.sessionPool) < cap(f.sessionPool) {
+				f.sessionPool <- session
+			}
+		}
+		if len(f.sessionPool) > 0 {
+			fmt.Printf("Added %d FlareSolverr sessions to the pool\n", len(f.sessionPool))
+		}
+	}
+
+	// If fewer than poolSize sessions were found, create new ones to fill the pool
+	for len(f.sessionPool) < cap(f.sessionPool) {
+		f.CreateSession()
+	}
+
+	return nil
+}
+
+func (f *FlareSolverr) CreateSession() string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	body := map[string]string{"cmd": "sessions.create"}
+	jsonBody, err := json.Marshal(body)
+	if err != nil {
+		return ""
+	}
+
+	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
+	if err != nil {
+		return ""
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := f.httpClient.Do(req)
+	if err != nil {
+		return ""
+	}
+
+	defer resp.Body.Close()
+
+	var sessionResponse map[string]interface{}
+	err = json.NewDecoder(resp.Body).Decode(&sessionResponse)
+	if err != nil {
+		return ""
+	}
+
+	session := sessionResponse["session"].(string)
+	// Add session to the pool
+	f.sessionPool <- session
+
+	fmt.Println("Created new FlareSolverr session:", session)
+	return session
+}
+
+func (f *FlareSolverr) ListSessions() ([]string, error) {
+	body := map[string]string{"cmd": "sessions.list"}
+	jsonBody, err := json.Marshal(body)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := f.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	var sessionsResponse map[string]interface{}
+	err = json.NewDecoder(resp.Body).Decode(&sessionsResponse)
+	if err != nil {
+		return nil, err
+	}
+	if sessionsResponse["sessions"] == nil {
+		return nil, ErrListSessions
+	}
+
+	sessions := sessionsResponse["sessions"].([]interface{})
+	var sessionIDs []string
+	for _, session := range sessions {
+		sessionIDs = append(sessionIDs, session.(string))
+	}
+
+	return sessionIDs, nil
+}
+
+func (f *FlareSolverr) RetrieveSession() string {
+	// Blocking receive from the session pool.
+	session := <-f.sessionPool
+	return session
+}
+
+type Response struct {
+	Status   string `json:"status"`
+	Message  string `json:"message"`
+	Solution struct {
+		Url     string `json:"url"`
+		Status  int    `json:"status"`
+		Cookies []struct {
+			Domain   string `json:"domain"`
+			Expiry   int    `json:"expiry"`
+			HttpOnly bool   `json:"httpOnly"`
+			Name     string `json:"name"`
+			Path     string `json:"path"`
+			SameSite string `json:"sameSite"`
+			Secure   bool   `json:"secure"`
+			Value    string `json:"value"`
+		} `json:"cookies"`
+		UserAgent string            `json:"userAgent"`
+		Headers   map[string]string `json:"headers"`
+		Response  string            `json:"response"`
+	} `json:"solution"`
+}
+
+func (f *FlareSolverr) Get(_url string) (io.ReadCloser, error) {
+	// Check if the FlareSolverr instance was initiated
+	if !f.initiated {
+		return io.NopCloser(bytes.NewReader([]byte(""))), nil
+	}
+
+	// Retrieve session from the pool (blocking if no sessions available)
+	session := f.RetrieveSession()
+
+	// Ensure the session is returned to the pool after the request is done
+	defer func() {
+		f.sessionPool <- session
+	}()
+
+	body := map[string]string{
+		"cmd":        "request.get",
+		"url":        _url,
+		"maxTimeout": fmt.Sprintf("%d", f.maxTimeout),
+		"session":    session,
+	}
+	jsonBody, err := json.Marshal(body)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := f.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse the response
+	var response Response
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if the response was successful
+	if response.Status != "ok" {
+		return nil, fmt.Errorf("failed to get response: %s", response.Message)
+	}
+
+	// Check if "Under attack" is in the response
+	if strings.Contains(response.Solution.Response, "Under attack") {
+		return nil, fmt.Errorf("under attack")
+	}
+
+	// check if the response is valid HTML
+	if !utils.IsValidHTML(response.Solution.Response) {
+		fmt.Printf("[FlareSolverr] Invalid HTML response from %s\n", _url)
+		response.Solution.Response = ""
+	}
+
+	// If the response body is empty but cookies are present, make a new request
+	if response.Solution.Response == "" && len(response.Solution.Cookies) > 0 {
+		fmt.Printf("[FlareSolverr] Making a new request to %s with cookies\n", _url)
+		// Create a new request with cookies
+		client := &http.Client{}
+		cookieJar, err := cookiejar.New(&cookiejar.Options{})
+		if err != nil {
+			return nil, err
+		}
+		for _, cookie := range response.Solution.Cookies {
+			cookieJar.SetCookies(&url.URL{Host: cookie.Domain}, []*http.Cookie{
+				{
+					Name:   cookie.Name,
+					Value:  cookie.Value,
+					Domain: cookie.Domain,
+					Path:   cookie.Path,
+				},
+			})
+		}
+		client.Jar = cookieJar
+
+		secondReq, err := http.NewRequest("GET", _url, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		// use the same user returned by the FlareSolverr
+		secondReq.Header.Set("User-Agent", response.Solution.UserAgent)
+
+		secondResp, err := client.Do(secondReq)
+		if err != nil {
+			return nil, err
+		}
+
+		respByte := new(bytes.Buffer)
+		_, err = respByte.ReadFrom(secondResp.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		// Return the body of the second request
+		return io.NopCloser(bytes.NewReader(respByte.Bytes())), nil
+	}
+
+	// Return the original response body
+	return io.NopCloser(bytes.NewReader([]byte(response.Solution.Response))), nil
+}

requester/requester.go

@@ -0,0 +1,98 @@
+package requester
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"time"
+
+	"github.com/felipemarinho97/torrent-indexer/cache"
+	"github.com/felipemarinho97/torrent-indexer/utils"
+)
+
+const (
+	cacheKey = "shortLivedCache"
+)
+
+var challangeRegex = regexp.MustCompile(`(?i)(just a moment|cf-chl-bypass|under attack)`)
+
+type Requster struct {
+	fs                        *FlareSolverr
+	c                         *cache.Redis
+	httpClient                *http.Client
+	shortLivedCacheExpiration time.Duration
+}
+
+func NewRequester(fs *FlareSolverr, c *cache.Redis) *Requster {
+	return &Requster{fs: fs, httpClient: &http.Client{}, c: c, shortLivedCacheExpiration: 30 * time.Minute}
+}
+
+func (i *Requster) SetShortLivedCacheExpiration(expiration time.Duration) {
+	i.shortLivedCacheExpiration = expiration
+}
+
+func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser, error) {
+	var body io.ReadCloser
+
+	// try request from short-lived cache
+	key := fmt.Sprintf("%s:%s", cacheKey, url)
+	bodyByte, err := i.c.Get(ctx, key)
+	if err == nil {
+		fmt.Printf("returning from short-lived cache: %s\n", url)
+		body = io.NopCloser(bytes.NewReader(bodyByte))
+		return body, nil
+	}
+
+	// try request with plain client
+	resp, err := i.httpClient.Get(url)
+	if err != nil {
+		// try request with flare solverr
+		body, err = i.fs.Get(url)
+		if err != nil {
+			return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
+		}
+	} else {
+		defer resp.Body.Close()
+		body = resp.Body
+	}
+
+	bodyByte, err = io.ReadAll(body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+	if hasChallange(bodyByte) {
+		// try request with flare solverr
+		body, err = i.fs.Get(url)
+		if err != nil {
+			return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
+		}
+		bodyByte, err = io.ReadAll(body)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read response body: %w", err)
+		}
+		fmt.Printf("request served from flaresolverr: %s\n", url)
+	} else {
+		fmt.Printf("request served from plain client: %s\n", url)
+	}
+
+	// save response to cache if it's not a challange, body is not empty and is valid HTML
+	if !hasChallange(bodyByte) && len(bodyByte) > 0 && utils.IsValidHTML(string(bodyByte)) {
+		err = i.c.SetWithExpiration(ctx, key, bodyByte, i.shortLivedCacheExpiration)
+		if err != nil {
+			fmt.Printf("failed to save response to cache: %v\n", err)
+		}
+		fmt.Printf("saved to cache: %s\n", url)
+	} else {
+		return nil, fmt.Errorf("response is a challange")
+	}
+
+	return io.NopCloser(bytes.NewReader(bodyByte)), nil
+}
+
+// hasChallange checks if the body contains a challange by regex matching
+func hasChallange(body []byte) bool {
+	return challangeRegex.Match(body)
+}

schema/audio.go

@@ -1,10 +1,14 @@
 package schema
 
+import "strings"
+
 type Audio string
 
 const (
 	AudioPortuguese  = "Português"
 	AudioPortuguese2 = "Portugues"
+	AudioPortuguese3 = "PT-BR"
+	AudioPortuguese4 = "Dublado"
 	AudioEnglish     = "Inglês"
 	AudioEnglish2    = "Ingles"
 	AudioSpanish     = "Espanhol"
@@ -21,6 +25,7 @@ const (
 	AudioMandarin3   = "Chines"
 	AudioRussian     = "Russo"
 	AudioSwedish     = "Sueco"
+	AudioSwedish2    = "Suéco"
 	AudioUkrainian   = "Ucraniano"
 	AudioPolish      = "Polaco"
 	AudioPolish2     = "Polonês"
@@ -28,11 +33,41 @@ const (
 	AudioThai        = "Tailandês"
 	AudioThai2       = "Tailandes"
 	AudioTurkish     = "Turco"
+	AudioHindi       = "Hindi"
+	AudioFarsi       = "Persa"
+	AudioFarsi2      = "Farsi"
+	AudioFarsi3      = "Iraniano"
+	AudioMalay       = "Malaio"
+	AudioDutch       = "Holandês"
+	AudioDutch2      = "Holandes"
+	AudioFinnish     = "Finlandês"
+	AudioFinnish2    = "Finlandes"
+	AudioDanish      = "Dinamarquês"
+	AudioDanish2     = "Dinamarques"
+	AudioNorwegian   = "Norueguês"
+	AudioNorwegian2  = "Noruegues"
+	AudioIcelandic   = "Islandês"
+	AudioIcelandic2  = "Islandes"
+	AudioGreek       = "Grego"
+	AudioArabic      = "Árabe"
+	AudioArabic2     = "Arabe"
+	AudioHebrew      = "Hebraico"
+	AudioVietnamese  = "Vietnamita"
+	AudioIndonesian  = "Indonésio"
+	AudioIndonesian2 = "Indonesio"
+	AudioFilipino    = "Filipino"
+	AudioBengali     = "Bengali"
+	AudioTamil       = "Tamil"
+	AudioTelugu      = "Telugu"
+	AudioGujarati    = "Gujarati"
+	AudioMarathi     = "Marathi"
 )
 
 var AudioList = []Audio{
 	AudioPortuguese,
 	AudioPortuguese2,
+	AudioPortuguese3,
+	AudioPortuguese4,
 	AudioEnglish,
 	AudioEnglish2,
 	AudioSpanish,
@@ -49,6 +84,7 @@ var AudioList = []Audio{
 	AudioMandarin3,
 	AudioRussian,
 	AudioSwedish,
+	AudioSwedish2,
 	AudioUkrainian,
 	AudioPolish,
 	AudioPolish2,
@@ -56,27 +92,59 @@ var AudioList = []Audio{
 	AudioThai,
 	AudioThai2,
 	AudioTurkish,
+	AudioHindi,
+	AudioFarsi,
+	AudioFarsi2,
+	AudioFarsi3,
+	AudioMalay,
+	AudioDutch,
+	AudioDutch2,
+	AudioFinnish,
+	AudioFinnish2,
+	AudioDanish,
+	AudioDanish2,
+	AudioNorwegian,
+	AudioNorwegian2,
+	AudioIcelandic,
+	AudioIcelandic2,
+	AudioGreek,
+	AudioArabic,
+	AudioArabic2,
+	AudioHebrew,
+	AudioVietnamese,
+	AudioIndonesian,
+	AudioIndonesian2,
+	AudioFilipino,
+	AudioBengali,
+	AudioTamil,
+	AudioTelugu,
+	AudioGujarati,
+	AudioMarathi,
 }
 
 func (a Audio) String() string {
-	return a.toISO639_2()
+	return a.toTag()
 }
 
 func GetAudioFromString(s string) *Audio {
 	for _, a := range AudioList {
-		if string(a) == s {
+		if strings.EqualFold(string(a), s) {
 			return &a
 		}
 	}
 	return nil
 }
 
-func (a Audio) toISO639_2() string {
+func (a Audio) toTag() string {
 	switch a {
 	case AudioPortuguese:
-		return "por"
+		return "brazilian"
 	case AudioPortuguese2:
-		return "por"
+		return "brazilian"
+	case AudioPortuguese3:
+		return "brazilian"
+	case AudioPortuguese4:
+		return "brazilian"
 	case AudioEnglish:
 		return "eng"
 	case AudioEnglish2:
@@ -109,6 +177,8 @@ func (a Audio) toISO639_2() string {
 		return "rus"
 	case AudioSwedish:
 		return "swe"
+	case AudioSwedish2:
+		return "swe"
 	case AudioUkrainian:
 		return "ukr"
 	case AudioPolish:
@@ -123,6 +193,62 @@ func (a Audio) toISO639_2() string {
 		return "tha"
 	case AudioTurkish:
 		return "tur"
+	case AudioHindi:
+		return "hin"
+	case AudioFarsi:
+		return "fas"
+	case AudioFarsi2:
+		return "fas"
+	case AudioFarsi3:
+		return "fas"
+	case AudioMalay:
+		return "msa"
+	case AudioDutch:
+		return "nld"
+	case AudioDutch2:
+		return "nld"
+	case AudioFinnish:
+		return "fin"
+	case AudioFinnish2:
+		return "fin"
+	case AudioDanish:
+		return "dan"
+	case AudioDanish2:
+		return "dan"
+	case AudioNorwegian:
+		return "nor"
+	case AudioNorwegian2:
+		return "nor"
+	case AudioIcelandic:
+		return "isl"
+	case AudioIcelandic2:
+		return "isl"
+	case AudioGreek:
+		return "ell"
+	case AudioArabic:
+		return "ara"
+	case AudioArabic2:
+		return "ara"
+	case AudioHebrew:
+		return "heb"
+	case AudioVietnamese:
+		return "vie"
+	case AudioIndonesian:
+		return "ind"
+	case AudioIndonesian2:
+		return "ind"
+	case AudioFilipino:
+		return "fil"
+	case AudioBengali:
+		return "ben"
+	case AudioTamil:
+		return "tam"
+	case AudioTelugu:
+		return "tel"
+	case AudioGujarati:
+		return "guj"
+	case AudioMarathi:
+		return "mar"
 	default:
 		return ""
 	}

schema/indexed_torrent.go

@@ -0,0 +1,26 @@
+package schema
+
+import "time"
+
+type IndexedTorrent struct {
+	Title         string    `json:"title"`
+	OriginalTitle string    `json:"original_title"`
+	Details       string    `json:"details"`
+	Year          string    `json:"year"`
+	IMDB          string    `json:"imdb"`
+	Audio         []Audio   `json:"audio"`
+	MagnetLink    string    `json:"magnet_link"`
+	Date          time.Time `json:"date"`
+	InfoHash      string    `json:"info_hash"`
+	Trackers      []string  `json:"trackers"`
+	Size          string    `json:"size"`
+	Files         []File    `json:"files,omitempty"`
+	LeechCount    int       `json:"leech_count"`
+	SeedCount     int       `json:"seed_count"`
+	Similarity    float32   `json:"similarity"`
+}
+
+type File struct {
+	Path string `json:"path"`
+	Size string `json:"size"`
+}

scrape/info.go

@@ -7,6 +7,8 @@ import (
 	"time"
 
 	"github.com/felipemarinho97/torrent-indexer/cache"
+	"github.com/felipemarinho97/torrent-indexer/monitoring"
+	"github.com/felipemarinho97/torrent-indexer/utils"
 )
 
 type peers struct {
@@ -44,19 +46,106 @@ func setPeersToCache(ctx context.Context, r *cache.Redis, infoHash string, peer,
 	return nil
 }
 
-func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, infoHash string, trackers []string) (int, int, error) {
+var additionalTrackers = []string{
+	"udp://tracker.opentrackr.org:1337/announce",
+	"udp://p4p.arenabg.com:1337/announce",
+	"udp://retracker.hotplug.ru:2710/announce",
+	"http://tracker.bt4g.com:2095/announce",
+	"http://bt.okmp3.ru:2710/announce",
+	"udp://tracker.torrent.eu.org:451/announce",
+	"http://tracker.mywaifu.best:6969/announce",
+	"udp://ttk2.nbaonlineservice.com:6969/announce",
+	"http://tracker.privateseedbox.xyz:2710/announce",
+	"udp://evan.im:6969/announce",
+	"https://tracker.yemekyedim.com:443/announce",
+	"udp://retracker.lanta.me:2710/announce",
+	"udp://martin-gebhardt.eu:25/announce",
+	"http://tracker.beeimg.com:6969/announce",
+	"udp://udp.tracker.projectk.org:23333/announce",
+	"http://tracker.renfei.net:8080/announce",
+	"https://tracker.expli.top:443/announce",
+	"https://tr.nyacat.pw:443/announce",
+	"udp://tracker.ducks.party:1984/announce",
+	"udp://extracker.dahrkael.net:6969/announce",
+	"http://ipv4.rer.lol:2710/announce",
+	"udp://tracker.plx.im:6969/announce",
+	"udp://tracker.tvunderground.org.ru:3218/announce",
+	"http://tracker.tricitytorrents.com:2710/announce",
+	"udp://open.stealth.si:80/announce",
+	"udp://tracker.dler.com:6969/announce",
+	"https://tracker.moeblog.cn:443/announce",
+	"udp://d40969.acod.regrucolo.ru:6969/announce",
+	"https://tracker.jdx3.org:443/announce",
+	"http://ipv6.rer.lol:6969/announce",
+	"udp://bandito.byterunner.io:6969/announce",
+	"udp://tracker.gigantino.net:6969/announce",
+	"http://tracker.netmap.top:6969/announce",
+	"udp://tracker.yume-hatsuyuki.moe:6969/announce",
+	"https://tracker.aburaya.live:443/announce",
+	"udp://tracker.srv00.com:6969/announce",
+	"udp://open.demonii.com:1337/announce",
+	"udp://1c.premierzal.ru:6969/announce",
+	"udp://tracker.fnix.net:6969/announce",
+	"udp://tracker.kmzs123.cn:17272/announce",
+	"https://tracker.home.kmzs123.cn:4443/announce",
+	"udp://tracker-udp.gbitt.info:80/announce",
+	"udp://tracker.torrust-demo.com:6969/announce",
+	"udp://tracker.hifimarket.in:2710/announce",
+	"udp://retracker01-msk-virt.corbina.net:80/announce",
+	"https://tracker.ghostchu-services.top:443/announce",
+	"udp://open.dstud.io:6969/announce",
+	"udp://tracker.therarbg.to:6969/announce",
+	"udp://tracker.bitcoinindia.space:6969/announce",
+	"udp://www.torrent.eu.org:451/announce",
+	"udp://tracker.hifitechindia.com:6969/announce",
+	"udp://tracker.gmi.gd:6969/announce",
+	"udp://tracker.skillindia.site:6969/announce",
+	"http://tracker.ipv6tracker.ru:80/announce",
+	"udp://tracker.tryhackx.org:6969/announce",
+	"http://torrent.hificode.in:6969/announce",
+	"http://open.trackerlist.xyz:80/announce",
+	"http://taciturn-shadow.spb.ru:6969/announce",
+	"http://0123456789nonexistent.com:80/announce",
+	"http://shubt.net:2710/announce",
+	"udp://tracker.valete.tf:9999/announce",
+	"https://tracker.zhuqiy.top:443/announce",
+	"https://tracker.leechshield.link:443/announce",
+	"http://tracker.tritan.gg:8080/announce",
+	"udp://t.overflow.biz:6969/announce",
+	"udp://open.tracker.cl:1337/announce",
+	"udp://explodie.org:6969/announce",
+	"udp://exodus.desync.com:6969/announce",
+	"udp://bt.ktrackers.com:6666/announce",
+	"udp://wepzone.net:6969/announce",
+	"udp://tracker2.dler.org:80/announce",
+	"udp://tracker.theoks.net:6969/announce",
+	"udp://tracker.ololosh.space:6969/announce",
+	"udp://tracker.filemail.com:6969/announce",
+	"udp://tracker.dump.cl:6969/announce",
+	"udp://tracker.dler.org:6969/announce",
+	"udp://tracker.bittor.pw:1337/announce",
+}
+
+func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metrics, infoHash string, trackers []string) (int, int, error) {
 	leech, seed, err := getPeersFromCache(ctx, r, infoHash)
 	if err != nil {
+		m.CacheHits.WithLabelValues("peers").Inc()
 		fmt.Println("unable to get peers from cache for infohash:", infoHash)
 	} else {
-		fmt.Println("get from cache> leech:", leech, "seed:", seed)
+		m.CacheMisses.WithLabelValues("peers").Inc()
+		fmt.Println("hash:", infoHash, "get from cache -> leech:", leech, "seed:", seed)
 		return leech, seed, nil
 	}
 
 	var peerChan = make(chan peers)
 	var errChan = make(chan error)
 
-	for _, tracker := range trackers {
+	allTrackers := make([]string, 0, len(trackers)+len(additionalTrackers))
+	allTrackers = append(allTrackers, trackers...)
+	allTrackers = append(allTrackers, additionalTrackers...)
+	allTrackers = utils.StableUniq(allTrackers)
+
+	for _, tracker := range allTrackers {
 		go func(tracker string) {
 			// get peers and seeds from redis first
 			scraper, err := New(tracker)
@@ -82,15 +171,20 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, infoHash string, tra
 	}
 
 	var peer peers
-	for i := 0; i < len(trackers); i++ {
+	for i := 0; i < len(allTrackers); i++ {
 		select {
+		case <-errChan:
+			// discard error
 		case peer = <-peerChan:
-			setPeersToCache(ctx, r, infoHash, peer.Leechers, peer.Seeders)
+			err = setPeersToCache(ctx, r, infoHash, peer.Leechers, peer.Seeders)
+			if err != nil {
+				fmt.Println(err)
+			} else {
+				fmt.Println("hash:", infoHash, "get from tracker -> leech:", peer.Leechers, "seed:", peer.Seeders)
+			}
 			return peer.Leechers, peer.Seeders, nil
-		case err := <-errChan:
-			fmt.Println(err)
 		}
 	}
 
-	return 0, 0, fmt.Errorf("unable to get peers from trackers")
+	return 0, 0, fmt.Errorf("unable to get peers from trackers for infohash: %s", infoHash)
 }

search/meilisearch.go

@@ -0,0 +1,223 @@
+package meilisearch
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/felipemarinho97/torrent-indexer/schema"
+)
+
+// SearchIndexer integrates with Meilisearch to index and search torrent items.
+type SearchIndexer struct {
+	Client    *http.Client
+	BaseURL   string
+	APIKey    string
+	IndexName string
+}
+
+// IndexStats represents statistics about the Meilisearch index
+type IndexStats struct {
+	NumberOfDocuments int64            `json:"numberOfDocuments"`
+	IsIndexing        bool             `json:"isIndexing"`
+	FieldDistribution map[string]int64 `json:"fieldDistribution"`
+}
+
+// HealthStatus represents the health status of Meilisearch
+type HealthStatus struct {
+	Status string `json:"status"`
+}
+
+// NewSearchIndexer creates a new instance of SearchIndexer.
+func NewSearchIndexer(baseURL, apiKey, indexName string) *SearchIndexer {
+	return &SearchIndexer{
+		Client:    &http.Client{Timeout: 10 * time.Second},
+		BaseURL:   baseURL,
+		APIKey:    apiKey,
+		IndexName: indexName,
+	}
+}
+
+// IndexTorrent indexes a single torrent item in Meilisearch.
+func (t *SearchIndexer) IndexTorrent(torrent schema.IndexedTorrent) error {
+	url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
+
+	torrentWithKey := struct {
+		Hash string `json:"id"`
+		schema.IndexedTorrent
+	}{
+		Hash:           torrent.InfoHash,
+		IndexedTorrent: torrent,
+	}
+
+	jsonData, err := json.Marshal(torrentWithKey)
+	if err != nil {
+		return fmt.Errorf("failed to marshal torrent data: %w", err)
+	}
+
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if t.APIKey != "" {
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
+	}
+
+	resp, err := t.Client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	return nil
+}
+
+func (t *SearchIndexer) IndexTorrents(torrents []schema.IndexedTorrent) error {
+	url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
+
+	torrentsWithKey := make([]struct {
+		Hash string `json:"id"`
+		schema.IndexedTorrent
+	}, 0, len(torrents))
+	for _, torrent := range torrents {
+		torrentWithKey := struct {
+			Hash string `json:"id"`
+			schema.IndexedTorrent
+		}{
+			Hash:           torrent.InfoHash,
+			IndexedTorrent: torrent,
+		}
+		torrentsWithKey = append(torrentsWithKey, torrentWithKey)
+	}
+
+	jsonData, err := json.Marshal(torrentsWithKey)
+	if err != nil {
+		return fmt.Errorf("failed to marshal torrent data: %w", err)
+	}
+
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if t.APIKey != "" {
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
+	}
+
+	resp, err := t.Client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	return nil
+}
+
+// SearchTorrent searches indexed torrents in Meilisearch based on the query.
+func (t *SearchIndexer) SearchTorrent(query string, limit int) ([]schema.IndexedTorrent, error) {
+	url := fmt.Sprintf("%s/indexes/%s/search", t.BaseURL, t.IndexName)
+	requestBody := map[string]string{
+		"q": query,
+	}
+	jsonData, err := json.Marshal(requestBody)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal search query: %w", err)
+	}
+
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if t.APIKey != "" {
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
+	}
+
+	resp, err := t.Client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, fmt.Errorf("search failed: %s", body)
+	}
+
+	var result struct {
+		Hits []schema.IndexedTorrent `json:"hits"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return nil, fmt.Errorf("failed to parse search response: %w", err)
+	}
+
+	return result.Hits, nil
+}
+
+// GetStats retrieves statistics about the Meilisearch index including document count.
+// This method can be used for health checks and monitoring.
+func (t *SearchIndexer) GetStats() (*IndexStats, error) {
+	url := fmt.Sprintf("%s/indexes/%s/stats", t.BaseURL, t.IndexName)
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	if t.APIKey != "" {
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
+	}
+
+	resp, err := t.Client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return nil, fmt.Errorf("failed to get stats: status %d, body: %s", resp.StatusCode, body)
+	}
+
+	var stats IndexStats
+	if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
+		return nil, fmt.Errorf("failed to parse stats response: %w", err)
+	}
+
+	return &stats, nil
+}
+
+// IsHealthy checks if Meilisearch is available and responsive.
+// Returns true if the service is healthy, false otherwise.
+func (t *SearchIndexer) IsHealthy() bool {
+	url := fmt.Sprintf("%s/health", t.BaseURL)
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return false
+	}
+
+	// Use a shorter timeout for health checks
+	client := &http.Client{Timeout: 5 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	return resp.StatusCode == http.StatusOK
+}
+
+// GetDocumentCount returns the number of indexed documents.
+// This is a convenience method that extracts just the document count from stats.
+func (t *SearchIndexer) GetDocumentCount() (int64, error) {
+	stats, err := t.GetStats()
+	if err != nil {
+		return 0, err
+	}
+	return stats.NumberOfDocuments, nil
+}

utils/decoder.go

@@ -0,0 +1,28 @@
+package utils
+
+import (
+	"encoding/base64"
+	"html"
+)
+
+func DecodeAdLink(encodedStr string) (string, error) {
+	reversed := reverseString(encodedStr)
+
+	decodedBytes, err := base64.StdEncoding.DecodeString(reversed)
+	if err != nil {
+		return "", err
+	}
+
+	htmlUnescaped := html.UnescapeString(string(decodedBytes))
+
+	return htmlUnescaped, nil
+}
+
+// Helper function to reverse a string
+func reverseString(s string) string {
+	runes := []rune(s)
+	for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
+		runes[i], runes[j] = runes[j], runes[i]
+	}
+	return string(runes)
+}

utils/util.go

@@ -0,0 +1,107 @@
+package utils
+
+import (
+	"fmt"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+// Filter filters a slice based on a predicate function.
+func Filter[A any](arr []A, f func(A) bool) []A {
+	var res []A
+	res = make([]A, 0)
+	for _, v := range arr {
+		if f(v) {
+			res = append(res, v)
+		}
+	}
+	return res
+}
+
+// ParallelFlatMap applies a function to each item in the iterable concurrently
+// and returns a slice of results. It can handle errors by passing an error handler function.
+func ParallelFlatMap[T any, R any](iterable []T, mapper func(item T) ([]R, error), errHandler ...func(error)) []R {
+	var itChan = make(chan []R)
+	var errChan = make(chan error)
+	mappedItems := []R{}
+	for _, link := range iterable {
+		go func(link T) {
+			items, err := mapper(link)
+			if err != nil {
+				errChan <- err
+			}
+			itChan <- items
+		}(link)
+	}
+
+	for range iterable {
+		select {
+		case items := <-itChan:
+			mappedItems = append(mappedItems, items...)
+		case err := <-errChan:
+			for _, handler := range errHandler {
+				handler(err)
+			}
+			if len(errHandler) == 0 {
+				fmt.Println(err)
+			}
+		}
+	}
+	return mappedItems
+}
+
+// StableUniq removes duplicates from a slice while maintaining the order of elements.
+func StableUniq(s []string) []string {
+	var uniq []map[string]interface{}
+	m := make(map[string]map[string]interface{})
+	for i, v := range s {
+		m[v] = map[string]interface{}{
+			"v": v,
+			"i": i,
+		}
+	}
+	// to order by index
+	for _, v := range m {
+		uniq = append(uniq, v)
+	}
+
+	// sort by index
+	for i := 0; i < len(uniq); i++ {
+		for j := i + 1; j < len(uniq); j++ {
+			if uniq[i]["i"].(int) > uniq[j]["i"].(int) {
+				uniq[i], uniq[j] = uniq[j], uniq[i]
+			}
+		}
+	}
+
+	// get only values
+	var uniqValues []string
+	for _, v := range uniq {
+		uniqValues = append(uniqValues, v["v"].(string))
+	}
+
+	return uniqValues
+}
+
+func IsValidHTML(input string) bool {
+	r := strings.NewReader(input)
+	_, err := html.Parse(r)
+	return err == nil
+}
+
+// FormatBytes formats a byte size into a human-readable string.
+// It converts bytes to KB, MB, or GB as appropriate.
+func FormatBytes(bytes int64) string {
+	if bytes < 1024 {
+		return fmt.Sprintf("%d B", bytes)
+	} else if bytes < 1024*1024 {
+		return fmt.Sprintf("%.2f KB", float64(bytes)/1024)
+	} else if bytes < 1024*1024*1024 {
+		return fmt.Sprintf("%.2f MB", float64(bytes)/(1024*1024))
+	} else if bytes < 1024*1024*1024*1024 {
+		return fmt.Sprintf("%.2f GB", float64(bytes)/(1024*1024*1024))
+	} else {
+		return fmt.Sprintf("%.2f TB", float64(bytes)/(1024*1024*1024*1024))
+	}
+}

utils/website.go

@@ -0,0 +1,97 @@
+package utils
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"sync"
+)
+
+var commonTLDs = []string{
+	".com",
+	".net",
+	".org",
+	".info",
+	".biz",
+	".co",
+	".io",
+	".xyz",
+	".me",
+	".tv",
+	".cc",
+	".us",
+	".online",
+	".site",
+	".la",
+	".se",
+	".to",
+}
+
+var commonSubdomains = []string{
+	"", // no prefix
+	"www.",
+}
+
+var commonWebsiteSLDs = []string{
+	"bludv",
+	"torrentdosfilmes",
+	"comando",
+	"comandotorrents",
+	"comandohds",
+	"redetorrent",
+	"torrenting",
+	"baixarfilmesdubladosviatorrent",
+	"hidratorrents",
+	"wolverdonfilmes",
+	"starckfilmes",
+	"rapidotorrents",
+	"sitedetorrents",
+	"vamostorrent",
+	"AZTORRENTS",
+}
+
+var websitePatterns = []string{
+	`\[\s*ACESSE\s+%s\s*\]`,
+	`\[?\s*%s(\s*\])?`,
+}
+
+var regexesOnce sync.Once
+var regexes []*regexp.Regexp
+
+func getRegexes() []*regexp.Regexp {
+	regexesOnce.Do(func() {
+		var websites strings.Builder
+		websites.WriteString("(?i)(")
+		for _, prefix := range commonSubdomains {
+			for _, name := range commonWebsiteSLDs {
+				for _, tld := range commonTLDs {
+					websites.WriteString(fmt.Sprintf("%s%s%s|", prefix, name, tld))
+				}
+			}
+		}
+		// remove the last pipe character
+		websites.WriteString(")")
+
+		websitesStr := websites.String()
+		websitesStr = strings.Replace(websitesStr, "|)", ")", 1)
+
+		for _, pattern := range websitePatterns {
+			regexes = append(regexes, regexp.MustCompile(fmt.Sprintf(pattern, websitesStr)))
+		}
+	})
+	return regexes
+}
+
+// RemoveKnownWebsites removes known website patterns from the title.
+// It uses a set of common prefixes, names, and TLDs to identify and remove
+// website references from the title.
+// It also removes any common patterns like "[ ACESSE bludv.com ]" or
+// "[ bludv.se ]" or "bludv.xyz".
+func RemoveKnownWebsites(title string) string {
+	regexes := getRegexes()
+	for _, re := range regexes {
+		title = re.ReplaceAllString(title, "")
+	}
+	title = strings.TrimSpace(title)
+	return title
+}