Compare commits
32 Commits
feat/imbd-
...
v0.10.0
| Author | SHA1 | Date | |
|---|---|---|---|
| a999f065da | |||
|
|
20b054f331 | ||
| 782f9d6b3f | |||
| 0b21e3b1e7 | |||
| 27ab075da1 | |||
| f4476024da | |||
|
|
472fedc565 | ||
|
|
6525b5b2b0 | ||
|
|
5bdd2cdd85 | ||
| 0f359ba44e | |||
|
|
bc53fbab1a | ||
| dc3cb3be92 | |||
| 88d6d506bf | |||
| 0a702d1893 | |||
| a6c3d3eed1 | |||
| 6c02f72e13 | |||
| 339db28d5a | |||
| 5034a11a66 | |||
| e994ee109d | |||
| a6977aec0d | |||
| a6a848b284 | |||
| ced533cd40 | |||
| 2812c203c9 | |||
| 98f1700b21 | |||
| 268ece5650 | |||
| 322bb34ebb | |||
| 0aa941a69f | |||
| f7fee26ac1 | |||
| e5e753fc80 | |||
| c09fd505ab | |||
| b1cece2743 | |||
| bfe26c91e3 |
26
.github/workflows/pipeline.dockerfile
vendored
Normal file
26
.github/workflows/pipeline.dockerfile
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
#####################################################
|
||||
### Copy platform specific binary
|
||||
FROM bash as copy-binary
|
||||
ARG TARGETPLATFORM
|
||||
|
||||
RUN echo "Target Platform = ${TARGETPLATFORM}"
|
||||
|
||||
COPY dist .
|
||||
RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then cp torrentindexer_linux_amd64_linux_amd64_v1/torrent-indexer /torrent-indexer; fi
|
||||
RUN if [ "$TARGETPLATFORM" = "linux/386" ]; then cp torrentindexer_linux_386_linux_386/torrent-indexer /torrent-indexer; fi
|
||||
RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then cp torrentindexer_linux_arm64_linux_arm64/torrent-indexer /torrent-indexer; fi
|
||||
RUN if [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then cp torrentindexer_linux_arm_linux_arm_6/torrent-indexer /torrent-indexer; fi
|
||||
RUN if [ "$TARGETPLATFORM" = "linux/arm/v7" ]; then cp torrentindexer_linux_arm_linux_arm_7/torrent-indexer /torrent-indexer; fi
|
||||
RUN chmod +x /torrent-indexer
|
||||
|
||||
|
||||
#####################################################
|
||||
### Build Final Image
|
||||
FROM alpine as release
|
||||
LABEL maintainer="felipevm97@gmail.com"
|
||||
|
||||
COPY --from=copy-binary /torrent-indexer /app/
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENTRYPOINT ["/app/torrent-indexer"]
|
||||
158
.github/workflows/pipeline.yml
vendored
Normal file
158
.github/workflows/pipeline.yml
vendored
Normal file
@@ -0,0 +1,158 @@
|
||||
name: "Pipeline: Test, Lint, Build"
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
tags:
|
||||
- "v*"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
go-lint:
|
||||
name: Lint Go code
|
||||
runs-on: ubuntu-latest
|
||||
container: golang:1.22
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Config workspace folder as trusted
|
||||
run: git config --global --add safe.directory $GITHUB_WORKSPACE; git describe --dirty --always --tags
|
||||
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v4
|
||||
with:
|
||||
version: latest
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
args: --timeout 2m
|
||||
|
||||
- run: go mod tidy
|
||||
- name: Verify no changes from go mod tidy
|
||||
run: |
|
||||
git status --porcelain
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
echo 'To fix this check, run "make format" and commit the changes'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
go:
|
||||
name: Test Go code
|
||||
runs-on: ubuntu-latest
|
||||
container: golang:1.22
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Config workspace folder as trusted
|
||||
run: git config --global --add safe.directory $GITHUB_WORKSPACE; git describe --dirty --always --tags
|
||||
|
||||
- name: Download dependencies
|
||||
if: steps.cache-go.outputs.cache-hit != 'true'
|
||||
continue-on-error: ${{contains(matrix.go_version, 'beta') || contains(matrix.go_version, 'rc')}}
|
||||
run: go mod download
|
||||
|
||||
- name: Test
|
||||
continue-on-error: ${{contains(matrix.go_version, 'beta') || contains(matrix.go_version, 'rc')}}
|
||||
run: go test -shuffle=on -race -cover ./... -v
|
||||
|
||||
binaries:
|
||||
name: Build binaries
|
||||
needs: [go, go-lint]
|
||||
runs-on: ubuntu-latest
|
||||
container: goreleaser/goreleaser:v1.25.1
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Config workspace folder as trusted
|
||||
run: git config --global --add safe.directory $GITHUB_WORKSPACE; git describe --dirty --always --tags
|
||||
|
||||
- name: Run GoReleaser - SNAPSHOT
|
||||
if: startsWith(github.ref, 'refs/tags/') != true
|
||||
run: goreleaser release --clean --skip=publish --snapshot
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Run GoReleaser - RELEASE
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
run: goreleaser release --clean
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: binaries
|
||||
path: |
|
||||
dist
|
||||
!dist/*.tar.gz
|
||||
!dist/*.zip
|
||||
retention-days: 7
|
||||
|
||||
docker:
|
||||
name: Build and publish Docker images
|
||||
needs: [binaries]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DOCKER_IMAGE: ${{secrets.DOCKER_IMAGE}}
|
||||
steps:
|
||||
- name: Set up QEMU
|
||||
id: qemu
|
||||
uses: docker/setup-qemu-action@v3
|
||||
if: env.DOCKER_IMAGE != ''
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
if: env.DOCKER_IMAGE != ''
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
if: env.DOCKER_IMAGE != ''
|
||||
|
||||
- uses: actions/download-artifact@v4
|
||||
if: env.DOCKER_IMAGE != ''
|
||||
with:
|
||||
name: binaries
|
||||
path: dist
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: env.DOCKER_IMAGE != ''
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
if: env.DOCKER_IMAGE != ''
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata for Docker
|
||||
if: env.DOCKER_IMAGE != ''
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
labels: |
|
||||
maintainer=felipemarinho97
|
||||
images: |
|
||||
name=${{secrets.DOCKER_IMAGE}}
|
||||
name=ghcr.io/${{ github.repository }}
|
||||
tags: |
|
||||
type=ref,event=pr
|
||||
type=semver,pattern={{version}}
|
||||
type=raw,value=develop,enable={{is_default_branch}}
|
||||
|
||||
- name: Build and Push
|
||||
if: env.DOCKER_IMAGE != ''
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: .github/workflows/pipeline.dockerfile
|
||||
platforms: linux/amd64,linux/386,linux/arm/v6,linux/arm/v7,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
4
.golangci.yml
Normal file
4
.golangci.yml
Normal file
@@ -0,0 +1,4 @@
|
||||
issues:
|
||||
exclude-files:
|
||||
- scrape.go
|
||||
- infohash.go
|
||||
133
.goreleaser.yml
Normal file
133
.goreleaser.yml
Normal file
@@ -0,0 +1,133 @@
|
||||
# GoReleaser config
|
||||
project_name: torrent-indexer
|
||||
|
||||
builds:
|
||||
- id: torrentindexer_linux_amd64
|
||||
env:
|
||||
- CGO_ENABLED=1
|
||||
goos:
|
||||
- linux
|
||||
goarch:
|
||||
- amd64
|
||||
flags:
|
||||
- -tags=netgo
|
||||
ldflags:
|
||||
- "-extldflags '-static -lz'"
|
||||
- -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
|
||||
|
||||
- id: torrentindexer_linux_386
|
||||
env:
|
||||
- CGO_ENABLED=1
|
||||
- PKG_CONFIG_PATH=/i386/lib/pkgconfig
|
||||
goos:
|
||||
- linux
|
||||
goarch:
|
||||
- "386"
|
||||
flags:
|
||||
- -tags=netgo
|
||||
ldflags:
|
||||
- "-extldflags '-static'"
|
||||
- -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
|
||||
|
||||
- id: torrentindexer_linux_arm
|
||||
env:
|
||||
- CGO_ENABLED=1
|
||||
- CC=arm-linux-gnueabi-gcc
|
||||
- CXX=arm-linux-gnueabi-g++
|
||||
- PKG_CONFIG_PATH=/arm/lib/pkgconfig
|
||||
goos:
|
||||
- linux
|
||||
goarch:
|
||||
- arm
|
||||
goarm:
|
||||
- "5"
|
||||
- "6"
|
||||
- "7"
|
||||
flags:
|
||||
- -tags=netgo
|
||||
ldflags:
|
||||
- "-extldflags '-static'"
|
||||
- -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
|
||||
|
||||
- id: torrentindexer_linux_arm64
|
||||
env:
|
||||
- CGO_ENABLED=1
|
||||
- CC=aarch64-linux-gnu-gcc
|
||||
- CXX=aarch64-linux-gnu-g++
|
||||
- PKG_CONFIG_PATH=/arm64/lib/pkgconfig
|
||||
goos:
|
||||
- linux
|
||||
goarch:
|
||||
- arm64
|
||||
flags:
|
||||
- -tags=netgo
|
||||
ldflags:
|
||||
- "-extldflags '-static'"
|
||||
- -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
|
||||
|
||||
- id: torrentindexer_windows_386
|
||||
env:
|
||||
- CGO_ENABLED=1
|
||||
- CC=i686-w64-mingw32-gcc
|
||||
- CXX=i686-w64-mingw32-g++
|
||||
- PKG_CONFIG_PATH=/mingw32/lib/pkgconfig
|
||||
goos:
|
||||
- windows
|
||||
goarch:
|
||||
- "386"
|
||||
flags:
|
||||
- -tags=netgo
|
||||
ldflags:
|
||||
- "-extldflags '-static'"
|
||||
- -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
|
||||
|
||||
- id: torrentindexer_windows_amd64
|
||||
env:
|
||||
- CGO_ENABLED=1
|
||||
- CC=x86_64-w64-mingw32-gcc
|
||||
- CXX=x86_64-w64-mingw32-g++
|
||||
- PKG_CONFIG_PATH=/mingw64/lib/pkgconfig
|
||||
goos:
|
||||
- windows
|
||||
goarch:
|
||||
- amd64
|
||||
flags:
|
||||
- -tags=netgo
|
||||
ldflags:
|
||||
- "-extldflags '-static'"
|
||||
- -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
|
||||
|
||||
- id: torrentindexer_darwin_amd64
|
||||
env:
|
||||
- CGO_ENABLED=1
|
||||
- CC=o64-clang
|
||||
- CXX=o64-clang++
|
||||
- PKG_CONFIG_PATH=/darwin/lib/pkgconfig
|
||||
goos:
|
||||
- darwin
|
||||
goarch:
|
||||
- amd64
|
||||
flags:
|
||||
- -tags=netgo
|
||||
ldflags:
|
||||
- -s -w -X github.com/felipemarinho97/torrent-indexer/consts.gitSha={{.ShortCommit}} -X github.com/felipemarinho97/torrent-indexer/consts.gitTag={{.Version}}
|
||||
|
||||
archives:
|
||||
- format_overrides:
|
||||
- goos: windows
|
||||
format: zip
|
||||
|
||||
checksum:
|
||||
name_template: "{{ .ProjectName }}_checksums.txt"
|
||||
|
||||
snapshot:
|
||||
name_template: "{{ .Tag }}-SNAPSHOT"
|
||||
|
||||
release:
|
||||
draft: true
|
||||
|
||||
changelog:
|
||||
# sort: asc
|
||||
filters:
|
||||
exclude:
|
||||
- "^docs:"
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.19 as builder
|
||||
FROM golang:1.22 as builder
|
||||
|
||||
WORKDIR /go/src/app
|
||||
COPY . .
|
||||
@@ -8,6 +8,7 @@ RUN go install -v ./...
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app .
|
||||
|
||||
FROM alpine:latest
|
||||
LABEL maintainer="felipevm97@gmail.com"
|
||||
|
||||
RUN apk --no-cache add ca-certificates
|
||||
|
||||
|
||||
621
LICENSE
Normal file
621
LICENSE
Normal file
@@ -0,0 +1,621 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU General Public License is a free, copyleft license for
|
||||
software and other kinds of works.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
the GNU General Public License is intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users. We, the Free Software Foundation, use the
|
||||
GNU General Public License for most of our software; it applies also to
|
||||
any other work released this way by its authors. You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to prevent others from denying you
|
||||
these rights or asking you to surrender the rights. Therefore, you have
|
||||
certain responsibilities if you distribute copies of the software, or if
|
||||
you modify it: responsibilities to respect the freedom of others.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must pass on to the recipients the same
|
||||
freedoms that you received. You must make sure that they, too, receive
|
||||
or can get the source code. And you must show them these terms so they
|
||||
know their rights.
|
||||
|
||||
Developers that use the GNU GPL protect your rights with two steps:
|
||||
(1) assert copyright on the software, and (2) offer you this License
|
||||
giving you legal permission to copy, distribute and/or modify it.
|
||||
|
||||
For the developers' and authors' protection, the GPL clearly explains
|
||||
that there is no warranty for this free software. For both users' and
|
||||
authors' sake, the GPL requires that modified versions be marked as
|
||||
changed, so that their problems will not be attributed erroneously to
|
||||
authors of previous versions.
|
||||
|
||||
Some devices are designed to deny users access to install or run
|
||||
modified versions of the software inside them, although the manufacturer
|
||||
can do so. This is fundamentally incompatible with the aim of
|
||||
protecting users' freedom to change the software. The systematic
|
||||
pattern of such abuse occurs in the area of products for individuals to
|
||||
use, which is precisely where it is most unacceptable. Therefore, we
|
||||
have designed this version of the GPL to prohibit the practice for those
|
||||
products. If such problems arise substantially in other domains, we
|
||||
stand ready to extend this provision to those domains in future versions
|
||||
of the GPL, as needed to protect the freedom of users.
|
||||
|
||||
Finally, every program is threatened constantly by software patents.
|
||||
States should not allow patents to restrict development and use of
|
||||
software on general-purpose computers, but in those that do, we wish to
|
||||
avoid the special danger that patents applied to a free program could
|
||||
make it effectively proprietary. To prevent this, the GPL assures that
|
||||
patents cannot be used to render the program non-free.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Use with the GNU Affero General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU Affero General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the special requirements of the GNU Affero General Public License,
|
||||
section 13, concerning interaction through a network will apply to the
|
||||
combination as such.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
9
Makefile
Normal file
9
Makefile
Normal file
@@ -0,0 +1,9 @@
|
||||
|
||||
build:
|
||||
docker build -t torrent-indexer .
|
||||
|
||||
lint:
|
||||
go run github.com/golangci/golangci-lint/cmd/golangci-lint@latest run -v --timeout 5m
|
||||
|
||||
run:
|
||||
go run main.go
|
||||
201
README.md
201
README.md
@@ -4,9 +4,208 @@ This is a simple torrent indexer that can be used to index torrents from HTML pa
|
||||
|
||||
## Test it
|
||||
|
||||
Visit [https://vlambdas.oci.darklyn.online/](https://vlambdas.oci.darklyn.online/) to test it.
|
||||
Visit [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org/) to test it.
|
||||
|
||||
## Supported sites
|
||||
|
||||
- [comando-torrents](https://comando.la/)
|
||||
- [bludv](https://bludvfilmes.tv/)
|
||||
- [torrent-dos-filmes](https://torrentdosfilmes.se/)
|
||||
- [starck-filmes](https://www.starckfilmes.online/)
|
||||
- [comandohds](https://comandohds.org/)
|
||||
- [rede-torrent](https://redetorrent.com/)
|
||||
|
||||
## Deploy
|
||||
|
||||
If you have Docker + docker-compose installed, you can deploy it using the following command:
|
||||
|
||||
```bash
|
||||
curl -s https://raw.githubusercontent.com/felipemarinho97/torrent-indexer/main/docker-compose.yml > docker-compose.yml
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
The server will be available at [http://localhost:8080/](http://localhost:8080/).
|
||||
|
||||
## Configuration
|
||||
|
||||
You can configure the server using the following environment variables:
|
||||
|
||||
- `PORT`: (optional) The port that the server will listen to. Default: `7006`
|
||||
- `FLARESOLVERR_ADDRESS`: (optional) The address of the FlareSolverr instance. Default: `N/A`
|
||||
- `MEILISEARCH_ADDRESS`: (optional) The address of the MeiliSearch instance. Default: `N/A`
|
||||
- `MEILISEARCH_KEY`: (optional) The API key of the MeiliSearch instance. Default: `N/A`
|
||||
- `REDIS_HOST`: (optional) The address of the Redis instance. Default: `localhost`
|
||||
- `SHORT_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the short-lived cache in duration format. Default: `30m`
|
||||
- This cache is used to cache homepage or search results.
|
||||
- Example: `30m`, `1h`, `1h30m`, `1h30m30s`
|
||||
- `LONG_LIVED_CACHE_EXPIRATION` (optional) The expiration time of the long-lived cache in duration format. Default: `7d`
|
||||
- This cache is used to store the torrent webpages (posts). You can set it to a higher value because the torrent pages are not updated frequently.
|
||||
|
||||
## Integrating with Jackett
|
||||
|
||||
You can integrate this indexer with Jackett by adding a new Torznab custom indexer. Here is an example of how to do it for the `bludv` indexer:
|
||||
|
||||
```yaml
|
||||
---
|
||||
id: bludv_indexer
|
||||
name: BluDV Indexer
|
||||
description: "BluDV - Custom indexer on from torrent-indexer"
|
||||
language: pt-BR
|
||||
type: public
|
||||
encoding: UTF-8
|
||||
links:
|
||||
- http://localhost:8080/
|
||||
|
||||
caps:
|
||||
categorymappings:
|
||||
- { id: Movie, cat: Movies, desc: "Movies" }
|
||||
- { id: TV, cat: TV, desc: "TV" }
|
||||
|
||||
modes:
|
||||
search: [q]
|
||||
tv-search: [q, season, ep]
|
||||
movie-search: [q]
|
||||
allowrawsearch: true
|
||||
|
||||
settings: []
|
||||
|
||||
search:
|
||||
paths:
|
||||
- path: "indexers/bludv?filter_results=true&q={{ .Keywords }}"
|
||||
response:
|
||||
type: json
|
||||
|
||||
keywordsfilters:
|
||||
- name: tolower
|
||||
|
||||
rows:
|
||||
selector: $.results
|
||||
count:
|
||||
selector: $.count
|
||||
|
||||
fields:
|
||||
_id:
|
||||
selector: title
|
||||
download:
|
||||
selector: magnet_link
|
||||
title:
|
||||
selector: title
|
||||
description:
|
||||
selector: original_title
|
||||
details:
|
||||
selector: details
|
||||
infohash:
|
||||
selector: info_hash
|
||||
date:
|
||||
selector: date
|
||||
size:
|
||||
selector: size
|
||||
seeders:
|
||||
selector: seed_count
|
||||
leechers:
|
||||
selector: leech_count
|
||||
imdb:
|
||||
selector: imdb
|
||||
category_is_tv_show:
|
||||
selector: title
|
||||
filters:
|
||||
- name: regexp
|
||||
args: "\\b(S\\d+(?:E\\d+)?)\\b"
|
||||
category:
|
||||
text: "{{ if .Result.category_is_tv_show }}TV{{ else }}Movie{{ end }}"
|
||||
# json engine n/a
|
||||
```
|
||||
|
||||
## Integrating with Prowlarr
|
||||
|
||||
You can integrate this indexer with Prowlarr by adding a custom definition. See [Adding a custom YML definition](https://wiki.servarr.com/prowlarr/indexers#adding-a-custom-yml-definition).
|
||||
|
||||
```yaml
|
||||
---
|
||||
id: torrent-indexer
|
||||
name: Torrent Indexer
|
||||
description: "Indexing Brazilian Torrent websites into structured data. github.com/felipemarinho97/torrent-indexer"
|
||||
language: pt-BR
|
||||
type: public
|
||||
encoding: UTF-8
|
||||
links:
|
||||
- http://localhost:8080/
|
||||
|
||||
caps:
|
||||
categories:
|
||||
Movies: Movies
|
||||
TV: TV
|
||||
|
||||
modes:
|
||||
search: [q]
|
||||
tv-search: [q, season]
|
||||
movie-search: [q]
|
||||
|
||||
settings:
|
||||
- name: indexer
|
||||
type: select
|
||||
label: Indexer
|
||||
default: bludv
|
||||
options:
|
||||
bludv: BLUDV
|
||||
comando_torrents: Comando Torrents
|
||||
torrent-dos-filmes: Torrent dos Filmes
|
||||
comandohds: Comando HDs
|
||||
starck-filmes: Starck Filmes
|
||||
rede_torrent: Rede Torrent
|
||||
|
||||
search:
|
||||
paths:
|
||||
- path: "/indexers/{{ .Config.indexer }}"
|
||||
response:
|
||||
type: json
|
||||
inputs:
|
||||
filter_results: "true"
|
||||
q: "{{ .Keywords }}"
|
||||
keywordsfilters:
|
||||
- name: tolower
|
||||
- name: re_replace
|
||||
args: ["(?i)(S0)(\\d{1,2})$", "temporada $2"]
|
||||
- name: re_replace
|
||||
args: ["(?i)(S)(\\d{1,3})$", "temporada $2"]
|
||||
|
||||
rows:
|
||||
selector: $.results
|
||||
count:
|
||||
selector: $.count
|
||||
|
||||
fields:
|
||||
download:
|
||||
selector: magnet_link
|
||||
title:
|
||||
selector: title
|
||||
description:
|
||||
selector: original_title
|
||||
details:
|
||||
selector: details
|
||||
infohash:
|
||||
selector: info_hash
|
||||
date:
|
||||
selector: date
|
||||
size:
|
||||
selector: size
|
||||
seeders:
|
||||
selector: seed_count
|
||||
leechers:
|
||||
selector: leech_count
|
||||
imdb:
|
||||
selector: imdb
|
||||
category_is_tv_show:
|
||||
selector: title
|
||||
filters:
|
||||
- name: regexp
|
||||
args: "\\b(S\\d+(?:E\\d+)?)\\b"
|
||||
category:
|
||||
text: "{{ if .Result.category_is_tv_show }}TV{{ else }}Movies{{ end }}"
|
||||
```
|
||||
|
||||
# Warning
|
||||
|
||||
The instance running at [https://torrent-indexer.darklyn.org/](https://torrent-indexer.darklyn.org/) is my personal instance and it is not guaranteed to be up all the time. Also, for better availability, I recommend deploying your own instance because the Cloudflare protection may block requests from indexed sites if too many requests are made in a short period of time from the same IP.
|
||||
|
||||
If I notice that the instance is being used a lot, I may block requests from Jackett to avoid overloading the server without prior notice.
|
||||
|
||||
161
api/bludv.go
161
api/bludv.go
@@ -6,46 +6,67 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/hbollon/go-edlib"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/magnet"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
)
|
||||
|
||||
var bludv = IndexerMeta{
|
||||
URL: "https://bludvfilmes.tv/",
|
||||
URL: "https://bludv.xyz/",
|
||||
SearchURL: "?s=",
|
||||
}
|
||||
|
||||
func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
i.metrics.IndexerDuration.WithLabelValues("bludv").Observe(time.Since(start).Seconds())
|
||||
i.metrics.IndexerRequests.WithLabelValues("bludv").Inc()
|
||||
}()
|
||||
|
||||
ctx := r.Context()
|
||||
// supported query params: q, season, episode
|
||||
// supported query params: q, season, episode, page, filter_results
|
||||
q := r.URL.Query().Get("q")
|
||||
page := r.URL.Query().Get("page")
|
||||
|
||||
// URL encode query param
|
||||
q = url.QueryEscape(q)
|
||||
url := bludv.URL
|
||||
if q != "" {
|
||||
if page != "" {
|
||||
url = fmt.Sprintf("%spage/%s", url, page)
|
||||
} else {
|
||||
url = fmt.Sprintf("%s%s%s", url, bludv.SearchURL, q)
|
||||
}
|
||||
|
||||
fmt.Println("URL:>", url)
|
||||
resp, err := http.Get(url)
|
||||
resp, err := i.requester.GetDocument(ctx, url)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("bludv").Inc()
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
defer resp.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
doc, err := goquery.NewDocumentFromReader(resp)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
i.metrics.IndexerErrors.WithLabelValues("bludv").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
@@ -56,9 +77,9 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
links = append(links, link)
|
||||
})
|
||||
|
||||
var itChan = make(chan []IndexedTorrent)
|
||||
var itChan = make(chan []schema.IndexedTorrent)
|
||||
var errChan = make(chan error)
|
||||
indexedTorrents := []IndexedTorrent{}
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
for _, link := range links {
|
||||
go func(link string) {
|
||||
torrents, err := getTorrentsBluDV(ctx, i, link)
|
||||
@@ -79,15 +100,42 @@ func (i *Indexer) HandlerBluDVIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
for i, it := range indexedTorrents {
|
||||
jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
|
||||
qLower := strings.ToLower(q)
|
||||
splitLength := 2
|
||||
indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
|
||||
}
|
||||
|
||||
// remove the ones with zero similarity
|
||||
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
|
||||
return it.Similarity > 0
|
||||
})
|
||||
}
|
||||
|
||||
// sort by similarity
|
||||
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
|
||||
return int((j.Similarity - i.Similarity) * 1000)
|
||||
})
|
||||
|
||||
// send to search index
|
||||
go func() {
|
||||
_ = i.search.IndexTorrents(indexedTorrents)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(Response{
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
Count: len(indexedTorrents),
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
|
||||
var indexedTorrents []IndexedTorrent
|
||||
func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
|
||||
var indexedTorrents []schema.IndexedTorrent
|
||||
doc, err := getDocument(ctx, i, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -96,22 +144,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
|
||||
article := doc.Find(".post")
|
||||
title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1)
|
||||
textContent := article.Find("div.content")
|
||||
// div itemprop="datePublished"
|
||||
datePublished := strings.TrimSpace(article.Find("div[itemprop=\"datePublished\"]").Text())
|
||||
// pattern: 10 de setembro de 2021
|
||||
re := regexp.MustCompile(`(\d{2}) de (\w+) de (\d{4})`)
|
||||
matches := re.FindStringSubmatch(datePublished)
|
||||
var date time.Time
|
||||
if len(matches) > 0 {
|
||||
day := matches[1]
|
||||
month := matches[2]
|
||||
year := matches[3]
|
||||
datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
|
||||
date, err = time.Parse("2006-01-02", datePublished)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
date := getPublishedDate(doc)
|
||||
magnets := textContent.Find("a[href^=\"magnet\"]")
|
||||
var magnetLinks []string
|
||||
magnets.Each(func(i int, s *goquery.Selection) {
|
||||
@@ -119,6 +152,30 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
|
||||
magnetLinks = append(magnetLinks, magnetLink)
|
||||
})
|
||||
|
||||
adwareLinks := textContent.Find("a[href^=\"https://www.seuvideo.xyz\"]")
|
||||
adwareLinks.Each(func(_ int, s *goquery.Selection) {
|
||||
href, _ := s.Attr("href")
|
||||
// extract querysting "id" from url
|
||||
parsedUrl, err := url.Parse(href)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
magnetLink := parsedUrl.Query().Get("id")
|
||||
magnetLinkDecoded, err := utils.DecodeAdLink(magnetLink)
|
||||
if err != nil {
|
||||
fmt.Printf("failed to decode ad link \"%s\": %v\n", href, err)
|
||||
return
|
||||
}
|
||||
|
||||
// if decoded magnet link is indeed a magnet link, append it
|
||||
if strings.HasPrefix(magnetLinkDecoded, "magnet:") {
|
||||
magnetLinks = append(magnetLinks, magnetLinkDecoded)
|
||||
} else {
|
||||
fmt.Printf("WARN: link \"%s\" decoding resulted in non-magnet link: %s\n", href, magnetLinkDecoded)
|
||||
}
|
||||
})
|
||||
|
||||
var audio []schema.Audio
|
||||
var year string
|
||||
var size []string
|
||||
@@ -142,7 +199,10 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
|
||||
text := s.Text()
|
||||
|
||||
audio = append(audio, findAudioFromText(text)...)
|
||||
year = findYearFromText(text, title)
|
||||
y := findYearFromText(text, title)
|
||||
if y != "" {
|
||||
year = y
|
||||
}
|
||||
size = append(size, findSizesFromText(text)...)
|
||||
})
|
||||
|
||||
@@ -150,16 +210,15 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
|
||||
imdbLink := ""
|
||||
article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Attr("href")
|
||||
re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
|
||||
matches := re.FindStringSubmatch(link)
|
||||
if len(matches) > 0 {
|
||||
imdbLink = matches[0]
|
||||
_imdbLink, err := getIMDBLink(link)
|
||||
if err == nil {
|
||||
imdbLink = _imdbLink
|
||||
}
|
||||
})
|
||||
|
||||
size = stableUniq(size)
|
||||
|
||||
var chanIndexedTorrent = make(chan IndexedTorrent)
|
||||
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
|
||||
|
||||
// for each magnet link, create a new indexed torrent
|
||||
for it, magnetLink := range magnetLinks {
|
||||
@@ -172,21 +231,9 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
|
||||
releaseTitle := magnet.DisplayName
|
||||
infoHash := magnet.InfoHash.String()
|
||||
trackers := magnet.Trackers
|
||||
magnetAudio := []schema.Audio{}
|
||||
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
|
||||
magnetAudio = append(magnetAudio, audio...)
|
||||
} else if len(audio) > 1 {
|
||||
// remove portuguese audio, and append to magnetAudio
|
||||
for _, a := range audio {
|
||||
if a != schema.AudioPortuguese {
|
||||
magnetAudio = append(magnetAudio, a)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
magnetAudio = append(magnetAudio, audio...)
|
||||
}
|
||||
magnetAudio := getAudioFromTitle(releaseTitle, audio)
|
||||
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, infoHash, trackers)
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
@@ -199,7 +246,7 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
|
||||
mySize = size[it]
|
||||
}
|
||||
|
||||
ixt := IndexedTorrent{
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Details: link,
|
||||
@@ -225,3 +272,15 @@ func getTorrentsBluDV(ctx context.Context, i *Indexer, link string) ([]IndexedTo
|
||||
|
||||
return indexedTorrents, nil
|
||||
}
|
||||
|
||||
func getPublishedDate(document *goquery.Document) time.Time {
|
||||
var date time.Time
|
||||
//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">
|
||||
datePublished := strings.TrimSpace(document.Find("meta[property=\"article:published_time\"]").AttrOr("content", ""))
|
||||
|
||||
if datePublished != "" {
|
||||
date, _ = time.Parse(time.RFC3339, datePublished)
|
||||
}
|
||||
|
||||
return date
|
||||
}
|
||||
|
||||
@@ -5,10 +5,11 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -16,6 +17,8 @@ import (
|
||||
"github.com/felipemarinho97/torrent-indexer/magnet"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
"github.com/hbollon/go-edlib"
|
||||
)
|
||||
|
||||
var comando = IndexerMeta{
|
||||
@@ -39,30 +42,47 @@ var replacer = strings.NewReplacer(
|
||||
)
|
||||
|
||||
func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
i.metrics.IndexerDuration.WithLabelValues("comando").Observe(time.Since(start).Seconds())
|
||||
i.metrics.IndexerRequests.WithLabelValues("comando").Inc()
|
||||
}()
|
||||
|
||||
ctx := r.Context()
|
||||
// supported query params: q, season, episode
|
||||
// supported query params: q, season, episode, page, filter_results
|
||||
q := r.URL.Query().Get("q")
|
||||
page := r.URL.Query().Get("page")
|
||||
|
||||
// URL encode query param
|
||||
q = url.QueryEscape(q)
|
||||
url := comando.URL
|
||||
if q != "" {
|
||||
url = fmt.Sprintf("%s%s%s", url, comando.SearchURL, q)
|
||||
} else if page != "" {
|
||||
url = fmt.Sprintf("%spage/%s", url, page)
|
||||
}
|
||||
|
||||
fmt.Println("URL:>", url)
|
||||
resp, err := http.Get(url)
|
||||
resp, err := i.requester.GetDocument(ctx, url)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("comando").Inc()
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
defer resp.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
doc, err := goquery.NewDocumentFromReader(resp)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("comando").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
@@ -73,9 +93,9 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
|
||||
links = append(links, link)
|
||||
})
|
||||
|
||||
var itChan = make(chan []IndexedTorrent)
|
||||
var itChan = make(chan []schema.IndexedTorrent)
|
||||
var errChan = make(chan error)
|
||||
indexedTorrents := []IndexedTorrent{}
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
for _, link := range links {
|
||||
go func(link string) {
|
||||
torrents, err := getTorrents(ctx, i, link)
|
||||
@@ -96,15 +116,42 @@ func (i *Indexer) HandlerComandoIndexer(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
for i, it := range indexedTorrents {
|
||||
jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
|
||||
qLower := strings.ToLower(q)
|
||||
splitLength := 2
|
||||
indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
|
||||
}
|
||||
|
||||
// remove the ones with zero similarity
|
||||
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
|
||||
return it.Similarity > 0
|
||||
})
|
||||
}
|
||||
|
||||
// sort by similarity
|
||||
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
|
||||
return int((j.Similarity - i.Similarity) * 1000)
|
||||
})
|
||||
|
||||
// send to search index
|
||||
go func() {
|
||||
_ = i.search.IndexTorrents(indexedTorrents)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(Response{
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
Count: len(indexedTorrents),
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent, error) {
|
||||
var indexedTorrents []IndexedTorrent
|
||||
func getTorrents(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
|
||||
var indexedTorrents []schema.IndexedTorrent
|
||||
doc, err := getDocument(ctx, i, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -116,19 +163,11 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
|
||||
// div itemprop="datePublished"
|
||||
datePublished := strings.TrimSpace(article.Find("div[itemprop=\"datePublished\"]").Text())
|
||||
// pattern: 10 de setembro de 2021
|
||||
re := regexp.MustCompile(`(\d{2}) de (\w+) de (\d{4})`)
|
||||
matches := re.FindStringSubmatch(datePublished)
|
||||
var date time.Time
|
||||
if len(matches) > 0 {
|
||||
day := matches[1]
|
||||
month := matches[2]
|
||||
year := matches[3]
|
||||
datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
|
||||
date, err = time.Parse("2006-01-02", datePublished)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
date, err := parseLocalizedDate(datePublished)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
magnets := textContent.Find("a[href^=\"magnet\"]")
|
||||
var magnetLinks []string
|
||||
magnets.Each(func(i int, s *goquery.Selection) {
|
||||
@@ -159,24 +198,26 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
|
||||
text := s.Text()
|
||||
|
||||
audio = append(audio, findAudioFromText(text)...)
|
||||
year = findYearFromText(text, title)
|
||||
y := findYearFromText(text, title)
|
||||
if y != "" {
|
||||
year = y
|
||||
}
|
||||
size = append(size, findSizesFromText(text)...)
|
||||
})
|
||||
|
||||
// find any link from imdb
|
||||
imdbLink := ""
|
||||
article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
|
||||
article.Find("a").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Attr("href")
|
||||
re := regexp.MustCompile(`https://www.imdb.com/title/(tt\d+)`)
|
||||
matches := re.FindStringSubmatch(link)
|
||||
if len(matches) > 0 {
|
||||
imdbLink = matches[0]
|
||||
_imdbLink, err := getIMDBLink(link)
|
||||
if err == nil {
|
||||
imdbLink = _imdbLink
|
||||
}
|
||||
})
|
||||
|
||||
size = stableUniq(size)
|
||||
|
||||
var chanIndexedTorrent = make(chan IndexedTorrent)
|
||||
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
|
||||
|
||||
// for each magnet link, create a new indexed torrent
|
||||
for it, magnetLink := range magnetLinks {
|
||||
@@ -189,21 +230,9 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
|
||||
releaseTitle := magnet.DisplayName
|
||||
infoHash := magnet.InfoHash.String()
|
||||
trackers := magnet.Trackers
|
||||
magnetAudio := []schema.Audio{}
|
||||
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
|
||||
magnetAudio = append(magnetAudio, audio...)
|
||||
} else if len(audio) > 1 {
|
||||
// remove portuguese audio, and append to magnetAudio
|
||||
for _, a := range audio {
|
||||
if a != schema.AudioPortuguese {
|
||||
magnetAudio = append(magnetAudio, a)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
magnetAudio = append(magnetAudio, audio...)
|
||||
}
|
||||
magnetAudio := getAudioFromTitle(releaseTitle, audio)
|
||||
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, infoHash, trackers)
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
@@ -216,7 +245,7 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
|
||||
mySize = size[it]
|
||||
}
|
||||
|
||||
ixt := IndexedTorrent{
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Details: link,
|
||||
@@ -243,6 +272,27 @@ func getTorrents(ctx context.Context, i *Indexer, link string) ([]IndexedTorrent
|
||||
return indexedTorrents, nil
|
||||
}
|
||||
|
||||
func parseLocalizedDate(datePublished string) (time.Time, error) {
|
||||
re := regexp.MustCompile(`(\d{1,2}) de (\w+) de (\d{4})`)
|
||||
matches := re.FindStringSubmatch(datePublished)
|
||||
if len(matches) > 0 {
|
||||
day := matches[1]
|
||||
// append 0 to single digit day
|
||||
if len(day) == 1 {
|
||||
day = fmt.Sprintf("0%s", day)
|
||||
}
|
||||
month := matches[2]
|
||||
year := matches[3]
|
||||
datePublished = fmt.Sprintf("%s-%s-%s", year, replacer.Replace(month), day)
|
||||
date, err := time.Parse("2006-01-02", datePublished)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return date, nil
|
||||
}
|
||||
return time.Time{}, nil
|
||||
}
|
||||
|
||||
func stableUniq(s []string) []string {
|
||||
var uniq []map[string]interface{}
|
||||
m := make(map[string]map[string]interface{})
|
||||
@@ -275,56 +325,6 @@ func stableUniq(s []string) []string {
|
||||
return uniqValues
|
||||
}
|
||||
|
||||
func findYearFromText(text string, title string) (year string) {
|
||||
re := regexp.MustCompile(`Lançamento: (.*)`)
|
||||
yearMatch := re.FindStringSubmatch(text)
|
||||
if len(yearMatch) > 0 {
|
||||
year = yearMatch[1]
|
||||
}
|
||||
|
||||
if year == "" {
|
||||
re = regexp.MustCompile(`\((\d{4})\)`)
|
||||
yearMatch := re.FindStringSubmatch(title)
|
||||
if len(yearMatch) > 0 {
|
||||
year = yearMatch[1]
|
||||
}
|
||||
}
|
||||
return year
|
||||
}
|
||||
|
||||
func findAudioFromText(text string) []schema.Audio {
|
||||
var audio []schema.Audio
|
||||
re := regexp.MustCompile(`(.udio|Idioma):.?(.*)`)
|
||||
audioMatch := re.FindStringSubmatch(text)
|
||||
if len(audioMatch) > 0 {
|
||||
sep := getSeparator(audioMatch[2])
|
||||
langs_raw := strings.Split(audioMatch[2], sep)
|
||||
for _, lang := range langs_raw {
|
||||
lang = strings.TrimSpace(lang)
|
||||
a := schema.GetAudioFromString(lang)
|
||||
if a != nil {
|
||||
audio = append(audio, *a)
|
||||
} else {
|
||||
fmt.Println("unknown language:", lang)
|
||||
}
|
||||
}
|
||||
}
|
||||
return audio
|
||||
}
|
||||
|
||||
func findSizesFromText(text string) []string {
|
||||
var sizes []string
|
||||
// everything that ends with GB or MB, using ',' or '.' as decimal separator
|
||||
re := regexp.MustCompile(`(\d+[\.,]?\d+) ?(GB|MB)`)
|
||||
sizesMatch := re.FindAllStringSubmatch(text, -1)
|
||||
if len(sizesMatch) > 0 {
|
||||
for _, size := range sizesMatch {
|
||||
sizes = append(sizes, size[0])
|
||||
}
|
||||
}
|
||||
return sizes
|
||||
}
|
||||
|
||||
func processTitle(title string, a []schema.Audio) string {
|
||||
// remove ' - Donwload' from title
|
||||
title = strings.Replace(title, " – Download", "", -1)
|
||||
@@ -338,40 +338,23 @@ func processTitle(title string, a []schema.Audio) string {
|
||||
return title
|
||||
}
|
||||
|
||||
func appendAudioISO639_2Code(title string, a []schema.Audio) string {
|
||||
if len(a) > 0 {
|
||||
audio := []string{}
|
||||
for _, lang := range a {
|
||||
audio = append(audio, lang.String())
|
||||
}
|
||||
title = fmt.Sprintf("%s (%s)", title, strings.Join(audio, ", "))
|
||||
}
|
||||
return title
|
||||
}
|
||||
|
||||
func getSeparator(s string) string {
|
||||
if strings.Contains(s, "|") {
|
||||
return "|"
|
||||
} else if strings.Contains(s, ",") {
|
||||
return ","
|
||||
}
|
||||
return " "
|
||||
}
|
||||
|
||||
func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Document, error) {
|
||||
// try to get from redis first
|
||||
docCache, err := i.redis.Get(ctx, link)
|
||||
if err == nil {
|
||||
return goquery.NewDocumentFromReader(ioutil.NopCloser(bytes.NewReader(docCache)))
|
||||
i.metrics.CacheHits.WithLabelValues("document_body").Inc()
|
||||
fmt.Printf("returning from long-lived cache: %s\n", link)
|
||||
return goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(docCache)))
|
||||
}
|
||||
defer i.metrics.CacheMisses.WithLabelValues("document_body").Inc()
|
||||
|
||||
resp, err := http.Get(link)
|
||||
resp, err := i.requester.GetDocument(ctx, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
defer resp.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
body, err := io.ReadAll(resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -382,7 +365,7 @@ func getDocument(ctx context.Context, i *Indexer, link string) (*goquery.Documen
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(ioutil.NopCloser(bytes.NewReader(body)))
|
||||
doc, err := goquery.NewDocumentFromReader(io.NopCloser(bytes.NewReader(body)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -3,76 +3,45 @@ package handler
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
"time"
|
||||
)
|
||||
|
||||
func Test_findAudioFromText(t *testing.T) {
|
||||
func Test_parseLocalizedDate(t *testing.T) {
|
||||
type args struct {
|
||||
text string
|
||||
datePublished string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want []schema.Audio
|
||||
name string
|
||||
args args
|
||||
want time.Time
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "should return audio in portuguese",
|
||||
name: "should return date",
|
||||
args: args{
|
||||
text: "Áudio: Português",
|
||||
},
|
||||
want: []schema.Audio{
|
||||
schema.AudioPortuguese,
|
||||
datePublished: "12 de outubro de 2022",
|
||||
},
|
||||
want: time.Date(2022, 10, 12, 0, 0, 0, 0, time.UTC),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "should return audio in portuguese",
|
||||
name: "should return date single digit",
|
||||
args: args{
|
||||
text: "Idioma: Português",
|
||||
},
|
||||
want: []schema.Audio{
|
||||
schema.AudioPortuguese,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "should return audio in portuguese",
|
||||
args: args{
|
||||
text: "Audio: Português",
|
||||
},
|
||||
want: []schema.Audio{
|
||||
schema.AudioPortuguese,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "should return audio in portuguese",
|
||||
args: args{
|
||||
text: `
|
||||
»INFORMAÇÕES«
|
||||
Título Traduzido: O Cangaceiro do Futuro
|
||||
Título Original: O Cangaceiro do Futuro
|
||||
IMDb: 7,1
|
||||
Gênero:Comédia
|
||||
Lançamento: 2022
|
||||
Qualidade: WEB-DL
|
||||
Áudio: Português
|
||||
Legenda: S/L
|
||||
Formato: MKV
|
||||
Tamanho: 5.77 GB | 9.60 GB
|
||||
Duração: 30 Min./Ep.
|
||||
Qualidade de Áudio: 10
|
||||
Qualidade de Vídeo: 10
|
||||
Servidor Via: Torrent
|
||||
`,
|
||||
},
|
||||
want: []schema.Audio{
|
||||
schema.AudioPortuguese,
|
||||
datePublished: "1 de outubro de 2022",
|
||||
},
|
||||
want: time.Date(2022, 10, 1, 0, 0, 0, 0, time.UTC),
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := findAudioFromText(tt.args.text); !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("findAudioFromText() = %v, want %v", got, tt.want)
|
||||
got, err := parseLocalizedDate(tt.args.datePublished)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("parseDate() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("parseDate() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
255
api/comandohds.go
Normal file
255
api/comandohds.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/hbollon/go-edlib"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/magnet"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
)
|
||||
|
||||
var comandohds = IndexerMeta{
|
||||
URL: "https://comandohds.org/",
|
||||
SearchURL: "?s=",
|
||||
}
|
||||
|
||||
var title_re = regexp.MustCompile(`^[(Filme)|(Série)\s]+`)
|
||||
|
||||
func (i *Indexer) HandlerComandoHDsIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
i.metrics.IndexerDuration.WithLabelValues("comandohds").Observe(time.Since(start).Seconds())
|
||||
i.metrics.IndexerRequests.WithLabelValues("comandohds").Inc()
|
||||
}()
|
||||
|
||||
ctx := r.Context()
|
||||
// supported query params: q, page, filter_results
|
||||
q := r.URL.Query().Get("q")
|
||||
page := r.URL.Query().Get("page")
|
||||
|
||||
// URL encode query param
|
||||
q = url.QueryEscape(q)
|
||||
url := comandohds.URL
|
||||
if q != "" {
|
||||
url = fmt.Sprintf("%s%s%s", url, comandohds.SearchURL, q)
|
||||
} else if page != "" {
|
||||
url = fmt.Sprintf("%spage/%s", url, page)
|
||||
}
|
||||
|
||||
fmt.Println("URL:>", url)
|
||||
resp, err := i.requester.GetDocument(ctx, url)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
|
||||
return
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
i.metrics.IndexerErrors.WithLabelValues("comandohds").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
var links []string
|
||||
doc.Find(".post").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Find("div.title > a").Attr("href")
|
||||
links = append(links, link)
|
||||
})
|
||||
|
||||
var itChan = make(chan []schema.IndexedTorrent)
|
||||
var errChan = make(chan error)
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
for _, link := range links {
|
||||
go func(link string) {
|
||||
torrents, err := getTorrentsComandoHDs(ctx, i, link)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
errChan <- err
|
||||
}
|
||||
itChan <- torrents
|
||||
}(link)
|
||||
}
|
||||
|
||||
for i := 0; i < len(links); i++ {
|
||||
select {
|
||||
case torrents := <-itChan:
|
||||
indexedTorrents = append(indexedTorrents, torrents...)
|
||||
case err := <-errChan:
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
for i, it := range indexedTorrents {
|
||||
jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
|
||||
qLower := strings.ToLower(q)
|
||||
splitLength := 2
|
||||
indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
|
||||
}
|
||||
|
||||
// remove the ones with zero similarity
|
||||
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
|
||||
return it.Similarity > 0
|
||||
})
|
||||
}
|
||||
|
||||
// sort by similarity
|
||||
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
|
||||
return int((j.Similarity - i.Similarity) * 1000)
|
||||
})
|
||||
|
||||
// send to search index
|
||||
go func() {
|
||||
_ = i.search.IndexTorrents(indexedTorrents)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
Count: len(indexedTorrents),
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getTorrentsComandoHDs(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
|
||||
var indexedTorrents []schema.IndexedTorrent
|
||||
doc, err := getDocument(ctx, i, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
article := doc.Find("article")
|
||||
title := title_re.ReplaceAllString(article.Find(".main_title > h1").Text(), "")
|
||||
textContent := article.Find("div.content")
|
||||
date := getPublishedDateFromMeta(doc)
|
||||
magnets := textContent.Find("a[href^=\"magnet\"]")
|
||||
var magnetLinks []string
|
||||
magnets.Each(func(i int, s *goquery.Selection) {
|
||||
magnetLink, _ := s.Attr("href")
|
||||
magnetLinks = append(magnetLinks, magnetLink)
|
||||
})
|
||||
|
||||
var audio []schema.Audio
|
||||
var year string
|
||||
var size []string
|
||||
article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
|
||||
// pattern:
|
||||
// »INFORMAÇÕES«
|
||||
// Titulo Traduzido: O Guerreiro Banido
|
||||
// Titulo Original: 天龍八部之喬峰傳
|
||||
// <picture />: 5.7
|
||||
// Ano de Lançamento: 2023
|
||||
// Gênero: Ação
|
||||
// Formato: MKV
|
||||
// Qualidade: WEB-DL
|
||||
// Idioma: Português | Inglês
|
||||
// Legenda: Português
|
||||
// Tamanho: – GB
|
||||
// Qualidade Áudio e Vídeo: 10
|
||||
// Duração: 130 Min
|
||||
// Servidor: Torrent
|
||||
text := s.Text()
|
||||
|
||||
audio = append(audio, findAudioFromText(text)...)
|
||||
y := findYearFromText(text, title)
|
||||
if y != "" {
|
||||
year = y
|
||||
}
|
||||
size = append(size, findSizesFromText(text)...)
|
||||
})
|
||||
|
||||
// find any link from imdb
|
||||
imdbLink := ""
|
||||
article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Attr("href")
|
||||
_imdbLink, err := getIMDBLink(link)
|
||||
if err == nil {
|
||||
imdbLink = _imdbLink
|
||||
}
|
||||
})
|
||||
|
||||
size = stableUniq(size)
|
||||
|
||||
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
|
||||
|
||||
// for each magnet link, create a new indexed torrent
|
||||
for it, magnetLink := range magnetLinks {
|
||||
it := it
|
||||
go func(it int, magnetLink string) {
|
||||
magnet, err := magnet.ParseMagnetUri(magnetLink)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
releaseTitle := strings.TrimSpace(magnet.DisplayName)
|
||||
infoHash := magnet.InfoHash.String()
|
||||
trackers := magnet.Trackers
|
||||
for i, tracker := range trackers {
|
||||
trackers[i] = strings.TrimSpace(tracker)
|
||||
}
|
||||
|
||||
magnetAudio := getAudioFromTitle(releaseTitle, audio)
|
||||
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
title := processTitle(title, magnetAudio)
|
||||
|
||||
// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
|
||||
var mySize string
|
||||
if len(size) == len(magnetLinks) {
|
||||
mySize = size[it]
|
||||
}
|
||||
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Details: link,
|
||||
Year: year,
|
||||
IMDB: imdbLink,
|
||||
Audio: magnetAudio,
|
||||
MagnetLink: magnetLink,
|
||||
Date: date,
|
||||
InfoHash: infoHash,
|
||||
Trackers: trackers,
|
||||
LeechCount: peer,
|
||||
SeedCount: seed,
|
||||
Size: mySize,
|
||||
}
|
||||
chanIndexedTorrent <- ixt
|
||||
}(it, magnetLink)
|
||||
}
|
||||
|
||||
for i := 0; i < len(magnetLinks); i++ {
|
||||
it := <-chanIndexedTorrent
|
||||
indexedTorrents = append(indexedTorrents, it)
|
||||
}
|
||||
|
||||
return indexedTorrents, nil
|
||||
}
|
||||
187
api/common.go
Normal file
187
api/common.go
Normal file
@@ -0,0 +1,187 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
)
|
||||
|
||||
func getPublishedDateFromMeta(document *goquery.Document) time.Time {
|
||||
var date time.Time
|
||||
//<meta property="article:published_time" content="2019-08-23T13:20:57+00:00">
|
||||
datePublished := strings.TrimSpace(document.Find("meta[property=\"article:published_time\"]").AttrOr("content", ""))
|
||||
|
||||
if datePublished != "" {
|
||||
date, _ = time.Parse(time.RFC3339, datePublished)
|
||||
}
|
||||
|
||||
return date
|
||||
}
|
||||
|
||||
type datePattern struct {
|
||||
regex *regexp.Regexp
|
||||
layout string
|
||||
}
|
||||
|
||||
var datePatterns = []datePattern{
|
||||
{regexp.MustCompile(`\d{4}-\d{2}-\d{2}`), "2006-01-02"},
|
||||
{regexp.MustCompile(`\d{2}-\d{2}-\d{4}`), "02-01-2006"},
|
||||
{regexp.MustCompile(`\d{2}/\d{2}/\d{4}`), "02/01/2006"},
|
||||
}
|
||||
|
||||
// getPublishedDateFromRawString extracts the date from a raw string using predefined patterns.
|
||||
func getPublishedDateFromRawString(dateStr string) time.Time {
|
||||
for _, p := range datePatterns {
|
||||
match := p.regex.FindString(dateStr)
|
||||
|
||||
if match != "" {
|
||||
date, err := time.Parse(p.layout, match)
|
||||
if err == nil {
|
||||
return date.UTC()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// getSeparator returns the separator used in the string.
|
||||
// It checks for common separators like "|", ",", "/", and " e "
|
||||
func getSeparator(s string) string {
|
||||
if strings.Contains(s, "|") {
|
||||
return "|"
|
||||
} else if strings.Contains(s, ",") {
|
||||
return ","
|
||||
} else if strings.Contains(s, "/") {
|
||||
return "/"
|
||||
} else if strings.Contains(s, " e ") {
|
||||
return " e "
|
||||
}
|
||||
return " "
|
||||
}
|
||||
|
||||
// findAudioFromText extracts audio languages from a given text.
|
||||
// It looks for patterns like "Áudio: Português, Inglês" or "Idioma: Português, Inglês"
|
||||
func findAudioFromText(text string) []schema.Audio {
|
||||
var audio []schema.Audio
|
||||
re := regexp.MustCompile(`(.udio|Idioma):.?(.*)`)
|
||||
audioMatch := re.FindStringSubmatch(text)
|
||||
if len(audioMatch) > 0 {
|
||||
sep := getSeparator(audioMatch[2])
|
||||
langs_raw := strings.Split(audioMatch[2], sep)
|
||||
for _, lang := range langs_raw {
|
||||
lang = strings.TrimSpace(lang)
|
||||
a := schema.GetAudioFromString(lang)
|
||||
if a != nil {
|
||||
audio = append(audio, *a)
|
||||
} else {
|
||||
fmt.Println("unknown language:", lang)
|
||||
}
|
||||
}
|
||||
}
|
||||
return audio
|
||||
}
|
||||
|
||||
// findYearFromText extracts the year from a given text.
|
||||
// It looks for patterns like "Lançamento: 2001" in the title.
|
||||
func findYearFromText(text string, title string) (year string) {
|
||||
re := regexp.MustCompile(`Lançamento: (.*)`)
|
||||
yearMatch := re.FindStringSubmatch(text)
|
||||
if len(yearMatch) > 0 {
|
||||
year = yearMatch[1]
|
||||
}
|
||||
|
||||
if year == "" {
|
||||
re = regexp.MustCompile(`\((\d{4})\)`)
|
||||
yearMatch := re.FindStringSubmatch(title)
|
||||
if len(yearMatch) > 0 {
|
||||
year = yearMatch[1]
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(year)
|
||||
}
|
||||
|
||||
// findSizesFromText extracts sizes from a given text.
|
||||
// It looks for patterns like "Tamanho: 1.26 GB" or "Tamanho: 700 MB".
|
||||
func findSizesFromText(text string) []string {
|
||||
var sizes []string
|
||||
// everything that ends with GB or MB, using ',' or '.' as decimal separator
|
||||
re := regexp.MustCompile(`(\d+[\.,]?\d+) ?(GB|MB)`)
|
||||
sizesMatch := re.FindAllStringSubmatch(text, -1)
|
||||
if len(sizesMatch) > 0 {
|
||||
for _, size := range sizesMatch {
|
||||
sizes = append(sizes, size[0])
|
||||
}
|
||||
}
|
||||
return sizes
|
||||
}
|
||||
|
||||
// getIMDBLink extracts the IMDB link from a given link.
|
||||
// It looks for patterns like "https://www.imdb.com/title/tt1234567/".
|
||||
// Returns an error if no valid IMDB link is found.
|
||||
func getIMDBLink(link string) (string, error) {
|
||||
var imdbLink string
|
||||
re := regexp.MustCompile(`https://www.imdb.com(/[a-z]{2})?/title/(tt\d+)/?`)
|
||||
|
||||
matches := re.FindStringSubmatch(link)
|
||||
if len(matches) > 0 {
|
||||
imdbLink = matches[0]
|
||||
} else {
|
||||
return "", fmt.Errorf("no imdb link found")
|
||||
}
|
||||
return imdbLink, nil
|
||||
}
|
||||
|
||||
// appendAudioISO639_2Code appends the audio languages to the title in ISO 639-2 code format.
|
||||
// It formats the title to include the audio languages in parentheses.
|
||||
// Example: "Movie Title (eng, por)"
|
||||
func appendAudioISO639_2Code(title string, a []schema.Audio) string {
|
||||
if len(a) > 0 {
|
||||
audio := []string{}
|
||||
for _, lang := range a {
|
||||
audio = append(audio, lang.String())
|
||||
}
|
||||
title = fmt.Sprintf("%s (%s)", title, strings.Join(audio, ", "))
|
||||
}
|
||||
return title
|
||||
}
|
||||
|
||||
// getAudioFromTitle extracts audio languages from the release title.
|
||||
// It checks for common patterns like "nacional", "dual", or "dublado"
|
||||
func getAudioFromTitle(releaseTitle string, audioFromContent []schema.Audio) []schema.Audio {
|
||||
magnetAudio := []schema.Audio{}
|
||||
isNacional := strings.Contains(strings.ToLower(releaseTitle), "nacional")
|
||||
if isNacional {
|
||||
magnetAudio = append(magnetAudio, schema.AudioPortuguese)
|
||||
}
|
||||
|
||||
if strings.Contains(strings.ToLower(releaseTitle), "dual") || strings.Contains(strings.ToLower(releaseTitle), "dublado") {
|
||||
magnetAudio = append(magnetAudio, audioFromContent...)
|
||||
// if Portuguese audio is not in the audio slice, append it
|
||||
if !slices.Contains(magnetAudio, schema.AudioPortuguese) {
|
||||
magnetAudio = append(magnetAudio, schema.AudioPortuguese)
|
||||
}
|
||||
} else if len(audioFromContent) > 1 {
|
||||
// remove portuguese audio, and append to magnetAudio
|
||||
for _, a := range audioFromContent {
|
||||
if a != schema.AudioPortuguese {
|
||||
magnetAudio = append(magnetAudio, a)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
magnetAudio = append(magnetAudio, audioFromContent...)
|
||||
}
|
||||
|
||||
// order and uniq the audio slice
|
||||
slices.SortFunc(magnetAudio, func(a, b schema.Audio) int {
|
||||
return strings.Compare(a.String(), b.String())
|
||||
})
|
||||
magnetAudio = slices.Compact(magnetAudio)
|
||||
|
||||
return magnetAudio
|
||||
}
|
||||
241
api/common_test.go
Normal file
241
api/common_test.go
Normal file
@@ -0,0 +1,241 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
)
|
||||
|
||||
func Test_getPublishedDateFromRawString(t *testing.T) {
|
||||
type args struct {
|
||||
dateStr string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want time.Time
|
||||
}{
|
||||
{
|
||||
name: "should parse date in format 2025-01-01",
|
||||
args: args{
|
||||
dateStr: "2025-01-01",
|
||||
},
|
||||
want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
name: "should parse date in format 01-01-2025",
|
||||
args: args{
|
||||
dateStr: "01-01-2025",
|
||||
},
|
||||
want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
name: "should parse date in format 01/01/2025",
|
||||
args: args{
|
||||
dateStr: "01/01/2025",
|
||||
},
|
||||
want: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
name: "should parse date from starck-filmes link",
|
||||
args: args{
|
||||
dateStr: "https://www.starckfilmes.online/catalog/jogos-de-seducao-2025-18-07-2025/",
|
||||
},
|
||||
want: time.Date(2025, 7, 18, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := getPublishedDateFromRawString(tt.args.dateStr); !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("getPublishedDateFromRawString() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_findAudioFromText(t *testing.T) {
|
||||
type args struct {
|
||||
text string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want []schema.Audio
|
||||
}{
|
||||
{
|
||||
name: "should return audio in portuguese",
|
||||
args: args{
|
||||
text: "Áudio: Português",
|
||||
},
|
||||
want: []schema.Audio{
|
||||
schema.AudioPortuguese,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "should return audio in portuguese",
|
||||
args: args{
|
||||
text: "Idioma: Português",
|
||||
},
|
||||
want: []schema.Audio{
|
||||
schema.AudioPortuguese,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "should return audio in portuguese",
|
||||
args: args{
|
||||
text: "Audio: Português",
|
||||
},
|
||||
want: []schema.Audio{
|
||||
schema.AudioPortuguese,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "should return audio in portuguese - comando_torrents",
|
||||
args: args{
|
||||
text: `
|
||||
»INFORMAÇÕES«
|
||||
Título Traduzido: O Cangaceiro do Futuro
|
||||
Título Original: O Cangaceiro do Futuro
|
||||
IMDb: 7,1
|
||||
Gênero:Comédia
|
||||
Lançamento: 2022
|
||||
Qualidade: WEB-DL
|
||||
Áudio: Português
|
||||
Legenda: S/L
|
||||
Formato: MKV
|
||||
Tamanho: 5.77 GB | 9.60 GB
|
||||
Duração: 30 Min./Ep.
|
||||
Qualidade de Áudio: 10
|
||||
Qualidade de Vídeo: 10
|
||||
Servidor Via: Torrent
|
||||
`,
|
||||
},
|
||||
want: []schema.Audio{
|
||||
schema.AudioPortuguese,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "should return audio in portuguese - rede torrent",
|
||||
args: args{
|
||||
text: `
|
||||
Filme Bicho de Sete Cabeças Torrent
|
||||
Título Original: Bicho de Sete Cabeças
|
||||
Lançamento: 2001
|
||||
Gêneros: Drama / Nacional
|
||||
Idioma: Português
|
||||
Qualidade: 720p / BluRay
|
||||
Duração: 1h 14 Minutos
|
||||
Formato: Mp4
|
||||
Vídeo: 10 e Áudio: 10
|
||||
Legendas: Português
|
||||
Nota do Imdb: 7.7
|
||||
Tamanho: 1.26 GB
|
||||
`,
|
||||
},
|
||||
want: []schema.Audio{
|
||||
schema.AudioPortuguese,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "should return audio in portuguese - rede torrent 2",
|
||||
args: args{
|
||||
text: `
|
||||
Filme Branca de Neve e o Caçador Torrent / Assistir Online
|
||||
Título Original: Snow White and the Huntsman
|
||||
Lançamento: 2012
|
||||
Gêneros: Ação / Aventura / Fantasia
|
||||
Idioma: Português / Inglês
|
||||
Duração: 126 Minutos
|
||||
Formato: Mkv / Mp4
|
||||
Vídeo: 10 e Áudio: 10
|
||||
Legendas: Sim
|
||||
Tamanho: 2.69 GB / 1.95 GB / 1.0 GB
|
||||
`,
|
||||
},
|
||||
want: []schema.Audio{
|
||||
schema.AudioPortuguese,
|
||||
schema.AudioEnglish,
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := findAudioFromText(tt.args.text); !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("findAudioFromText() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_getIMDBLink(t *testing.T) {
|
||||
type args struct {
|
||||
link string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "should return imdb link",
|
||||
args: args{
|
||||
link: "https://www.imdb.com/title/tt1234567",
|
||||
},
|
||||
want: "https://www.imdb.com/title/tt1234567",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "should return imdb link when end with /",
|
||||
args: args{
|
||||
link: "https://www.imdb.com/title/tt1234567/",
|
||||
},
|
||||
want: "https://www.imdb.com/title/tt1234567/",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "should return imdb link when end with /",
|
||||
args: args{
|
||||
link: "https://www.imdb.com/title/tt1234567/",
|
||||
},
|
||||
want: "https://www.imdb.com/title/tt1234567/",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "should return imdb link when it has a language",
|
||||
args: args{
|
||||
link: "https://www.imdb.com/pt/title/tt18722864/",
|
||||
},
|
||||
want: "https://www.imdb.com/pt/title/tt18722864/",
|
||||
},
|
||||
{
|
||||
name: "should return imdb link when it has a language",
|
||||
args: args{
|
||||
link: "https://www.imdb.com/pt/title/tt34608980/",
|
||||
},
|
||||
want: "https://www.imdb.com/pt/title/tt34608980/",
|
||||
},
|
||||
{
|
||||
name: "should return error when link is invalid",
|
||||
args: args{
|
||||
link: "https://www.google.com",
|
||||
},
|
||||
want: "",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := getIMDBLink(tt.args.link)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("getIMDBLink() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("getIMDBLink() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
140
api/index.go
140
api/index.go
@@ -6,42 +6,37 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/cache"
|
||||
"github.com/felipemarinho97/torrent-indexer/monitoring"
|
||||
"github.com/felipemarinho97/torrent-indexer/requester"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
meilisearch "github.com/felipemarinho97/torrent-indexer/search"
|
||||
)
|
||||
|
||||
type Indexer struct {
|
||||
redis *cache.Redis
|
||||
redis *cache.Redis
|
||||
metrics *monitoring.Metrics
|
||||
requester *requester.Requster
|
||||
search *meilisearch.SearchIndexer
|
||||
}
|
||||
|
||||
type IndexerMeta struct {
|
||||
URL string
|
||||
SearchURL string
|
||||
// pattern for pagination, e.g. "page/%s"
|
||||
PagePattern string
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
Results []IndexedTorrent `json:"results"`
|
||||
Count int `json:"count"`
|
||||
Results []schema.IndexedTorrent `json:"results"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
type IndexedTorrent struct {
|
||||
Title string `json:"title"`
|
||||
OriginalTitle string `json:"original_title"`
|
||||
Details string `json:"details"`
|
||||
Year string `json:"year"`
|
||||
IMDB string `json:"imdb"`
|
||||
Audio []schema.Audio `json:"audio"`
|
||||
MagnetLink string `json:"magnet_link"`
|
||||
Date time.Time `json:"date"`
|
||||
InfoHash string `json:"info_hash"`
|
||||
Trackers []string `json:"trackers"`
|
||||
Size string `json:"size"`
|
||||
LeechCount int `json:"leech_count"`
|
||||
SeedCount int `json:"seed_count"`
|
||||
}
|
||||
|
||||
func NewIndexers(redis *cache.Redis) *Indexer {
|
||||
func NewIndexers(redis *cache.Redis, metrics *monitoring.Metrics, req *requester.Requster, si *meilisearch.SearchIndexer) *Indexer {
|
||||
return &Indexer{
|
||||
redis: redis,
|
||||
redis: redis,
|
||||
metrics: metrics,
|
||||
requester: req,
|
||||
search: si,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,23 +44,104 @@ func HandlerIndex(w http.ResponseWriter, r *http.Request) {
|
||||
currentTime := time.Now().Format(time.RFC850)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
err := json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"time": currentTime,
|
||||
"endpoints": map[string]interface{}{
|
||||
"/indexers/comando_torrents": map[string]interface{}{
|
||||
"method": "GET",
|
||||
"description": "Indexer for comando torrents",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"/indexers/comando_torrents": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"description": "Indexer for comando torrents",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"page": "page number",
|
||||
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
|
||||
},
|
||||
},
|
||||
},
|
||||
"/indexers/bludv": map[string]interface{}{
|
||||
"method": "GET",
|
||||
"description": "Indexer for bludv",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"/indexers/bludv": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"description": "Indexer for bludv",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"page": "page number",
|
||||
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
|
||||
}},
|
||||
},
|
||||
"/indexers/torrent-dos-filmes": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"page": "page number",
|
||||
"description": "Indexer for Torrent dos Filmes",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
|
||||
},
|
||||
},
|
||||
},
|
||||
"/indexers/comandohds": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"page": "page number",
|
||||
"description": "Indexer for Comando HDs",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
|
||||
},
|
||||
},
|
||||
},
|
||||
"/indexers/starck-filmes": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"page": "page number",
|
||||
"description": "Indexer for Starck Filmes",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
|
||||
},
|
||||
},
|
||||
},
|
||||
"/indexers/rede_torrent": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"description": "Indexer for rede torrent",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
"page": "page number",
|
||||
"filter_results": "if results with similarity equals to zero should be filtered (true/false)",
|
||||
},
|
||||
},
|
||||
},
|
||||
"/indexers/manual": []map[string]interface{}{
|
||||
{
|
||||
"method": "POST",
|
||||
"description": "Add a manual torrent entry to the indexer for 12 hours",
|
||||
"body": map[string]interface{}{
|
||||
"magnetLink": "magnet link",
|
||||
}},
|
||||
{
|
||||
"method": "GET",
|
||||
"description": "Get all manual torrents",
|
||||
},
|
||||
},
|
||||
"/search": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"description": "Search for cached torrents across all indexers",
|
||||
"query_params": map[string]string{
|
||||
"q": "search query",
|
||||
},
|
||||
},
|
||||
},
|
||||
"/ui/": []map[string]interface{}{
|
||||
{
|
||||
"method": "GET",
|
||||
"description": "Show the unified search UI (only work if Meilisearch is enabled)",
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
125
api/manual.go
Normal file
125
api/manual.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/magnet"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const manualTorrentsRedisKey = "manual:torrents"
|
||||
|
||||
var manualTorrentExpiration = 8 * time.Hour
|
||||
|
||||
type ManualIndexerRequest struct {
|
||||
MagnetLink string `json:"magnetLink"`
|
||||
}
|
||||
|
||||
func (i *Indexer) HandlerManualIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
var req ManualIndexerRequest
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
|
||||
// fetch from redis
|
||||
out, err := i.redis.Get(ctx, manualTorrentsRedisKey)
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
fmt.Println(err)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
|
||||
return
|
||||
} else if errors.Is(err, redis.Nil) {
|
||||
out = bytes.NewBufferString("[]").Bytes()
|
||||
}
|
||||
|
||||
err = json.Unmarshal([]byte(out), &indexedTorrents)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
// check if the request is a POST
|
||||
if r.Method == http.MethodPost {
|
||||
// decode the request body
|
||||
err := json.NewDecoder(r.Body).Decode(&req)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
magnet, err := magnet.ParseMagnetUri(req.MagnetLink)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
var audio []schema.Audio
|
||||
releaseTitle := magnet.DisplayName
|
||||
infoHash := magnet.InfoHash.String()
|
||||
trackers := magnet.Trackers
|
||||
magnetAudio := getAudioFromTitle(releaseTitle, audio)
|
||||
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
title := processTitle(releaseTitle, magnetAudio)
|
||||
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Audio: magnetAudio,
|
||||
MagnetLink: req.MagnetLink,
|
||||
InfoHash: infoHash,
|
||||
Trackers: trackers,
|
||||
LeechCount: peer,
|
||||
SeedCount: seed,
|
||||
}
|
||||
|
||||
// write to redis
|
||||
indexedTorrents = append(indexedTorrents, ixt)
|
||||
out, err := json.Marshal(indexedTorrents)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
err = i.redis.SetWithExpiration(ctx, manualTorrentsRedisKey, out, manualTorrentExpiration)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("manual").Inc()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
Count: len(indexedTorrents),
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
280
api/rede_torrent.go
Normal file
280
api/rede_torrent.go
Normal file
@@ -0,0 +1,280 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/hbollon/go-edlib"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/magnet"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
)
|
||||
|
||||
var rede_torrent = IndexerMeta{
|
||||
URL: "https://redetorrent.com/",
|
||||
SearchURL: "index.php?s=",
|
||||
PagePattern: "%s",
|
||||
}
|
||||
|
||||
func (i *Indexer) HandlerRedeTorrentIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
i.metrics.IndexerDuration.WithLabelValues("rede_torrent").Observe(time.Since(start).Seconds())
|
||||
i.metrics.IndexerRequests.WithLabelValues("rede_torrent").Inc()
|
||||
}()
|
||||
|
||||
ctx := r.Context()
|
||||
// supported query params: q, season, episode, page, filter_results
|
||||
q := r.URL.Query().Get("q")
|
||||
page := r.URL.Query().Get("page")
|
||||
|
||||
// URL encode query param
|
||||
q = url.QueryEscape(q)
|
||||
url := rede_torrent.URL
|
||||
if q != "" {
|
||||
url = fmt.Sprintf("%s%s%s", url, rede_torrent.SearchURL, q)
|
||||
} else if page != "" {
|
||||
url = fmt.Sprintf(fmt.Sprintf("%s%s", url, rede_torrent.PagePattern), page)
|
||||
}
|
||||
|
||||
fmt.Println("URL:>", url)
|
||||
resp, err := i.requester.GetDocument(ctx, url)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("rede_torrent").Inc()
|
||||
return
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
i.metrics.IndexerErrors.WithLabelValues("rede_torrent").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
var links []string
|
||||
doc.Find(".capa_lista").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Find("a").Attr("href")
|
||||
links = append(links, link)
|
||||
})
|
||||
|
||||
var itChan = make(chan []schema.IndexedTorrent)
|
||||
var errChan = make(chan error)
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
for _, link := range links {
|
||||
go func(link string) {
|
||||
torrents, err := getTorrentsRedeTorrent(ctx, i, link)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
errChan <- err
|
||||
}
|
||||
itChan <- torrents
|
||||
}(link)
|
||||
}
|
||||
|
||||
for i := 0; i < len(links); i++ {
|
||||
select {
|
||||
case torrents := <-itChan:
|
||||
indexedTorrents = append(indexedTorrents, torrents...)
|
||||
case err := <-errChan:
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
for i, it := range indexedTorrents {
|
||||
jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
|
||||
qLower := strings.ToLower(q)
|
||||
splitLength := 2
|
||||
indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
|
||||
}
|
||||
|
||||
// remove the ones with zero similarity
|
||||
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
|
||||
return it.Similarity > 0
|
||||
})
|
||||
}
|
||||
|
||||
// sort by similarity
|
||||
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
|
||||
return int((j.Similarity - i.Similarity) * 1000)
|
||||
})
|
||||
|
||||
// send to search index
|
||||
go func() {
|
||||
_ = i.search.IndexTorrents(indexedTorrents)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
Count: len(indexedTorrents),
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getTorrentsRedeTorrent(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
|
||||
var indexedTorrents []schema.IndexedTorrent
|
||||
doc, err := getDocument(ctx, i, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
article := doc.Find(".conteudo")
|
||||
// title pattern: "Something - optional balbla (dddd) some shit" - extract "Something" and "dddd"
|
||||
titleRe := regexp.MustCompile(`^(.*?)(?: - (.*?))? \((\d{4})\)`)
|
||||
titleP := titleRe.FindStringSubmatch(article.Find("h1").Text())
|
||||
if len(titleP) < 3 {
|
||||
return nil, fmt.Errorf("could not extract title from %s", link)
|
||||
}
|
||||
title := strings.TrimSpace(titleP[1])
|
||||
year := strings.TrimSpace(titleP[3])
|
||||
|
||||
textContent := article.Find(".apenas_itemprop")
|
||||
date := getPublishedDateFromMeta(doc)
|
||||
magnets := textContent.Find("a[href^=\"magnet\"]")
|
||||
var magnetLinks []string
|
||||
magnets.Each(func(i int, s *goquery.Selection) {
|
||||
magnetLink, _ := s.Attr("href")
|
||||
magnetLinks = append(magnetLinks, magnetLink)
|
||||
})
|
||||
|
||||
var audio []schema.Audio
|
||||
var size []string
|
||||
article.Find("div#informacoes > p").Each(func(i int, s *goquery.Selection) {
|
||||
// pattern:
|
||||
// Filme Bicho de Sete Cabeças Torrent
|
||||
// Título Original: Bicho de Sete Cabeças
|
||||
// Lançamento: 2001
|
||||
// Gêneros: Drama / Nacional
|
||||
// Idioma: Português
|
||||
// Qualidade: 720p / BluRay
|
||||
// Duração: 1h 14 Minutos
|
||||
// Formato: Mp4
|
||||
// Vídeo: 10 e Áudio: 10
|
||||
// Legendas: Português
|
||||
// Nota do Imdb: 7.7
|
||||
// Tamanho: 1.26 GB
|
||||
|
||||
// we need to manualy parse because the text is not well formatted
|
||||
htmlContent, err := s.Html()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
// remove any \n and \t characters
|
||||
htmlContent = strings.ReplaceAll(htmlContent, "\n", "")
|
||||
htmlContent = strings.ReplaceAll(htmlContent, "\t", "")
|
||||
|
||||
// split by <br> tags and render each line
|
||||
brRe := regexp.MustCompile(`<br\s*\/?>`)
|
||||
htmlContent = brRe.ReplaceAllString(htmlContent, "<br>")
|
||||
lines := strings.Split(htmlContent, "<br>")
|
||||
|
||||
var text strings.Builder
|
||||
for _, line := range lines {
|
||||
// remove any HTML tags
|
||||
re := regexp.MustCompile(`<[^>]*>`)
|
||||
line = re.ReplaceAllString(line, "")
|
||||
|
||||
line = strings.TrimSpace(line)
|
||||
text.WriteString(line + "\n")
|
||||
}
|
||||
|
||||
audio = append(audio, findAudioFromText(text.String())...)
|
||||
y := findYearFromText(text.String(), title)
|
||||
if y != "" {
|
||||
year = y
|
||||
}
|
||||
size = append(size, findSizesFromText(text.String())...)
|
||||
})
|
||||
|
||||
// find any link from imdb
|
||||
imdbLink := ""
|
||||
article.Find("a").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Attr("href")
|
||||
_imdbLink, err := getIMDBLink(link)
|
||||
if err == nil {
|
||||
imdbLink = _imdbLink
|
||||
}
|
||||
})
|
||||
|
||||
size = stableUniq(size)
|
||||
|
||||
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
|
||||
|
||||
// for each magnet link, create a new indexed torrent
|
||||
for it, magnetLink := range magnetLinks {
|
||||
it := it
|
||||
go func(it int, magnetLink string) {
|
||||
magnet, err := magnet.ParseMagnetUri(magnetLink)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
releaseTitle := magnet.DisplayName
|
||||
infoHash := magnet.InfoHash.String()
|
||||
trackers := magnet.Trackers
|
||||
magnetAudio := getAudioFromTitle(releaseTitle, audio)
|
||||
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
title := processTitle(title, magnetAudio)
|
||||
|
||||
// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
|
||||
var mySize string
|
||||
if len(size) == len(magnetLinks) {
|
||||
mySize = size[it]
|
||||
}
|
||||
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Details: link,
|
||||
Year: year,
|
||||
IMDB: imdbLink,
|
||||
Audio: magnetAudio,
|
||||
MagnetLink: magnetLink,
|
||||
Date: date,
|
||||
InfoHash: infoHash,
|
||||
Trackers: trackers,
|
||||
LeechCount: peer,
|
||||
SeedCount: seed,
|
||||
Size: mySize,
|
||||
}
|
||||
chanIndexedTorrent <- ixt
|
||||
}(it, magnetLink)
|
||||
}
|
||||
|
||||
for i := 0; i < len(magnetLinks); i++ {
|
||||
it := <-chanIndexedTorrent
|
||||
indexedTorrents = append(indexedTorrents, it)
|
||||
}
|
||||
|
||||
return indexedTorrents, nil
|
||||
}
|
||||
158
api/search.go
Normal file
158
api/search.go
Normal file
@@ -0,0 +1,158 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
meilisearch "github.com/felipemarinho97/torrent-indexer/search"
|
||||
)
|
||||
|
||||
// MeilisearchHandler handles HTTP requests for Meilisearch integration.
|
||||
type MeilisearchHandler struct {
|
||||
Module *meilisearch.SearchIndexer
|
||||
}
|
||||
|
||||
// HealthResponse represents the health check response
|
||||
type HealthResponse struct {
|
||||
Status string `json:"status"`
|
||||
Service string `json:"service"`
|
||||
Details map[string]interface{} `json:"details,omitempty"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
}
|
||||
|
||||
// StatsResponse represents the stats endpoint response
|
||||
type StatsResponse struct {
|
||||
Status string `json:"status"`
|
||||
NumberOfDocuments int64 `json:"numberOfDocuments"`
|
||||
IsIndexing bool `json:"isIndexing"`
|
||||
FieldDistribution map[string]int64 `json:"fieldDistribution"`
|
||||
Service string `json:"service"`
|
||||
}
|
||||
|
||||
// NewMeilisearchHandler creates a new instance of MeilisearchHandler.
|
||||
func NewMeilisearchHandler(module *meilisearch.SearchIndexer) *MeilisearchHandler {
|
||||
return &MeilisearchHandler{Module: module}
|
||||
}
|
||||
|
||||
// SearchTorrentHandler handles the searching of torrent items.
|
||||
func (h *MeilisearchHandler) SearchTorrentHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
query := r.URL.Query().Get("q")
|
||||
if query == "" {
|
||||
http.Error(w, "Query parameter 'q' is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
limitStr := r.URL.Query().Get("limit")
|
||||
limit := 10 // Default limit
|
||||
if limitStr != "" {
|
||||
var err error
|
||||
limit, err = strconv.Atoi(limitStr)
|
||||
if err != nil || limit <= 0 {
|
||||
http.Error(w, "Invalid limit parameter", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
results, err := h.Module.SearchTorrent(query, limit)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to search torrents", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(results); err != nil {
|
||||
http.Error(w, "Failed to encode response", http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
// HealthHandler provides a health check endpoint for Meilisearch.
|
||||
func (h *MeilisearchHandler) HealthHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
// Check if Meilisearch is healthy
|
||||
isHealthy := h.Module.IsHealthy()
|
||||
|
||||
response := HealthResponse{
|
||||
Service: "meilisearch",
|
||||
Timestamp: getCurrentTimestamp(),
|
||||
}
|
||||
|
||||
if isHealthy {
|
||||
// Try to get additional stats for more detailed health info
|
||||
stats, err := h.Module.GetStats()
|
||||
if err == nil {
|
||||
response.Status = "healthy"
|
||||
response.Details = map[string]interface{}{
|
||||
"documents": stats.NumberOfDocuments,
|
||||
"indexing": stats.IsIndexing,
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
} else {
|
||||
// Service is up but can't get stats
|
||||
response.Status = "degraded"
|
||||
response.Details = map[string]interface{}{
|
||||
"error": "Could not retrieve stats",
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
} else {
|
||||
// Service is down
|
||||
response.Status = "unhealthy"
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}
|
||||
|
||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||||
http.Error(w, "Failed to encode response", http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
// StatsHandler provides detailed statistics about the Meilisearch index.
|
||||
func (h *MeilisearchHandler) StatsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
// Get detailed stats from Meilisearch
|
||||
stats, err := h.Module.GetStats()
|
||||
if err != nil {
|
||||
// Check if it's a connectivity issue
|
||||
if !h.Module.IsHealthy() {
|
||||
http.Error(w, "Meilisearch service is unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
http.Error(w, "Failed to retrieve statistics", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
response := StatsResponse{
|
||||
Status: "healthy",
|
||||
Service: "meilisearch",
|
||||
NumberOfDocuments: stats.NumberOfDocuments,
|
||||
IsIndexing: stats.IsIndexing,
|
||||
FieldDistribution: stats.FieldDistribution,
|
||||
}
|
||||
|
||||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||||
http.Error(w, "Failed to encode response", http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
// getCurrentTimestamp returns the current timestamp in RFC3339 format
|
||||
func getCurrentTimestamp() string {
|
||||
return time.Now().Format(time.RFC3339)
|
||||
}
|
||||
255
api/starck_filmes.go
Normal file
255
api/starck_filmes.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/hbollon/go-edlib"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/magnet"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
)
|
||||
|
||||
var starck_filmes = IndexerMeta{
|
||||
URL: "https://www.starckfilmes.online/",
|
||||
SearchURL: "?s=",
|
||||
}
|
||||
|
||||
func (i *Indexer) HandlerStarckFilmesIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
i.metrics.IndexerDuration.WithLabelValues("starck_filmes").Observe(time.Since(start).Seconds())
|
||||
i.metrics.IndexerRequests.WithLabelValues("starck_filmes").Inc()
|
||||
}()
|
||||
|
||||
ctx := r.Context()
|
||||
// supported query params: q, page, filter_results
|
||||
q := r.URL.Query().Get("q")
|
||||
page := r.URL.Query().Get("page")
|
||||
|
||||
// URL encode query param
|
||||
q = url.QueryEscape(q)
|
||||
url := starck_filmes.URL
|
||||
if q != "" {
|
||||
url = fmt.Sprintf("%s%s%s", url, starck_filmes.SearchURL, q)
|
||||
} else if page != "" {
|
||||
url = fmt.Sprintf("%spage/%s", url, page)
|
||||
}
|
||||
|
||||
fmt.Println("URL:>", url)
|
||||
resp, err := i.requester.GetDocument(ctx, url)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
|
||||
return
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
i.metrics.IndexerErrors.WithLabelValues("starck_filmes").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
var links []string
|
||||
doc.Find(".item").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Find("div.sub-item > a").Attr("href")
|
||||
links = append(links, link)
|
||||
})
|
||||
|
||||
var itChan = make(chan []schema.IndexedTorrent)
|
||||
var errChan = make(chan error)
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
for _, link := range links {
|
||||
go func(link string) {
|
||||
torrents, err := getTorrentStarckFilmes(ctx, i, link)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
errChan <- err
|
||||
}
|
||||
itChan <- torrents
|
||||
}(link)
|
||||
}
|
||||
|
||||
for i := 0; i < len(links); i++ {
|
||||
select {
|
||||
case torrents := <-itChan:
|
||||
indexedTorrents = append(indexedTorrents, torrents...)
|
||||
case err := <-errChan:
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
for i, it := range indexedTorrents {
|
||||
jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
|
||||
qLower := strings.ToLower(q)
|
||||
splitLength := 2
|
||||
indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
|
||||
}
|
||||
|
||||
// remove the ones with zero similarity
|
||||
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
|
||||
return it.Similarity > 0
|
||||
})
|
||||
}
|
||||
|
||||
// sort by similarity
|
||||
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
|
||||
return int((j.Similarity - i.Similarity) * 1000)
|
||||
})
|
||||
|
||||
// send to search index
|
||||
go func() {
|
||||
_ = i.search.IndexTorrents(indexedTorrents)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
Count: len(indexedTorrents),
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getTorrentStarckFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
|
||||
var indexedTorrents []schema.IndexedTorrent
|
||||
doc, err := getDocument(ctx, i, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
date := getPublishedDateFromRawString(link)
|
||||
|
||||
post := doc.Find(".post")
|
||||
capa := post.Find(".capa")
|
||||
title := capa.Find(".post-description > h2").Text()
|
||||
post_buttons := post.Find(".post-buttons")
|
||||
magnets := post_buttons.Find("a[href^=\"magnet\"]")
|
||||
var magnetLinks []string
|
||||
magnets.Each(func(i int, s *goquery.Selection) {
|
||||
magnetLink, _ := s.Attr("href")
|
||||
magnetLinks = append(magnetLinks, magnetLink)
|
||||
})
|
||||
|
||||
var audio []schema.Audio
|
||||
var year string
|
||||
var size []string
|
||||
capa.Find(".post-description p").Each(func(i int, s *goquery.Selection) {
|
||||
// pattern:
|
||||
// Nome Original: 28 Weeks Later
|
||||
// Lançamento: 2007
|
||||
// Duração: 1h 40 min
|
||||
// Gênero: Terror, Suspense, Ficção
|
||||
// Formato: MKV
|
||||
// Tamanho: 2.45 GB
|
||||
// Qualidade de Video: 10
|
||||
// Qualidade do Audio: 10
|
||||
// Idioma: Português | Inglês
|
||||
// Legenda: Português, Inglês, Espanhol
|
||||
var text strings.Builder
|
||||
s.Find("span").Each(func(i int, span *goquery.Selection) {
|
||||
text.WriteString(span.Text())
|
||||
text.WriteString(" ")
|
||||
})
|
||||
audio = append(audio, findAudioFromText(text.String())...)
|
||||
y := findYearFromText(text.String(), title)
|
||||
if y != "" {
|
||||
year = y
|
||||
}
|
||||
size = append(size, findSizesFromText(text.String())...)
|
||||
})
|
||||
|
||||
// TODO: find any link from imdb
|
||||
imdbLink := ""
|
||||
|
||||
size = stableUniq(size)
|
||||
|
||||
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
|
||||
|
||||
// for each magnet link, create a new indexed torrent
|
||||
for it, magnetLink := range magnetLinks {
|
||||
it := it
|
||||
go func(it int, magnetLink string) {
|
||||
magnet, err := magnet.ParseMagnetUri(magnetLink)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
releaseTitle := strings.TrimSpace(magnet.DisplayName)
|
||||
// url decode the title
|
||||
releaseTitle, err = url.QueryUnescape(releaseTitle)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
releaseTitle = strings.TrimSpace(magnet.DisplayName)
|
||||
}
|
||||
infoHash := magnet.InfoHash.String()
|
||||
trackers := magnet.Trackers
|
||||
for i, tracker := range trackers {
|
||||
unescapedTracker, err := url.QueryUnescape(tracker)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
trackers[i] = strings.TrimSpace(unescapedTracker)
|
||||
}
|
||||
magnetAudio := getAudioFromTitle(releaseTitle, audio)
|
||||
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
title := processTitle(title, magnetAudio)
|
||||
|
||||
// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
|
||||
var mySize string
|
||||
if len(size) == len(magnetLinks) {
|
||||
mySize = size[it]
|
||||
}
|
||||
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Details: link,
|
||||
Year: year,
|
||||
IMDB: imdbLink,
|
||||
Audio: magnetAudio,
|
||||
MagnetLink: magnetLink,
|
||||
Date: date,
|
||||
InfoHash: infoHash,
|
||||
Trackers: trackers,
|
||||
LeechCount: peer,
|
||||
SeedCount: seed,
|
||||
Size: mySize,
|
||||
}
|
||||
chanIndexedTorrent <- ixt
|
||||
}(it, magnetLink)
|
||||
}
|
||||
|
||||
for i := 0; i < len(magnetLinks); i++ {
|
||||
it := <-chanIndexedTorrent
|
||||
indexedTorrents = append(indexedTorrents, it)
|
||||
}
|
||||
|
||||
return indexedTorrents, nil
|
||||
}
|
||||
249
api/torrent_dos_filmes.go
Normal file
249
api/torrent_dos_filmes.go
Normal file
@@ -0,0 +1,249 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/hbollon/go-edlib"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/magnet"
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
goscrape "github.com/felipemarinho97/torrent-indexer/scrape"
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
)
|
||||
|
||||
var torrent_dos_filmes = IndexerMeta{
|
||||
URL: "https://torrentdosfilmes.se/",
|
||||
SearchURL: "?s=",
|
||||
}
|
||||
|
||||
func (i *Indexer) HandlerTorrentDosFilmesIndexer(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
i.metrics.IndexerDuration.WithLabelValues("torrent_dos_filmes").Observe(time.Since(start).Seconds())
|
||||
i.metrics.IndexerRequests.WithLabelValues("torrent_dos_filmes").Inc()
|
||||
}()
|
||||
|
||||
ctx := r.Context()
|
||||
// supported query params: q, season, episode, page, filter_results
|
||||
q := r.URL.Query().Get("q")
|
||||
page := r.URL.Query().Get("page")
|
||||
|
||||
// URL encode query param
|
||||
q = url.QueryEscape(q)
|
||||
url := torrent_dos_filmes.URL
|
||||
if q != "" {
|
||||
url = fmt.Sprintf("%s%s%s", url, torrent_dos_filmes.SearchURL, q)
|
||||
} else if page != "" {
|
||||
url = fmt.Sprintf("%spage/%s", url, page)
|
||||
}
|
||||
|
||||
fmt.Println("URL:>", url)
|
||||
resp, err := i.requester.GetDocument(ctx, url)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
i.metrics.IndexerErrors.WithLabelValues("torrent_dos_filmes").Inc()
|
||||
return
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
err = json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
i.metrics.IndexerErrors.WithLabelValues("torrent_dos_filmes").Inc()
|
||||
return
|
||||
}
|
||||
|
||||
var links []string
|
||||
doc.Find(".post").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Find("div.title > a").Attr("href")
|
||||
links = append(links, link)
|
||||
})
|
||||
|
||||
var itChan = make(chan []schema.IndexedTorrent)
|
||||
var errChan = make(chan error)
|
||||
indexedTorrents := []schema.IndexedTorrent{}
|
||||
for _, link := range links {
|
||||
go func(link string) {
|
||||
torrents, err := getTorrentsTorrentDosFilmes(ctx, i, link)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
errChan <- err
|
||||
}
|
||||
itChan <- torrents
|
||||
}(link)
|
||||
}
|
||||
|
||||
for i := 0; i < len(links); i++ {
|
||||
select {
|
||||
case torrents := <-itChan:
|
||||
indexedTorrents = append(indexedTorrents, torrents...)
|
||||
case err := <-errChan:
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
for i, it := range indexedTorrents {
|
||||
jLower := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("%s %s", it.Title, it.OriginalTitle)), ".", " ")
|
||||
qLower := strings.ToLower(q)
|
||||
splitLength := 2
|
||||
indexedTorrents[i].Similarity = edlib.JaccardSimilarity(jLower, qLower, splitLength)
|
||||
}
|
||||
|
||||
// remove the ones with zero similarity
|
||||
if len(indexedTorrents) > 20 && r.URL.Query().Get("filter_results") != "" && r.URL.Query().Get("q") != "" {
|
||||
indexedTorrents = utils.Filter(indexedTorrents, func(it schema.IndexedTorrent) bool {
|
||||
return it.Similarity > 0
|
||||
})
|
||||
}
|
||||
|
||||
// sort by similarity
|
||||
slices.SortFunc(indexedTorrents, func(i, j schema.IndexedTorrent) int {
|
||||
return int((j.Similarity - i.Similarity) * 1000)
|
||||
})
|
||||
|
||||
// send to search index
|
||||
go func() {
|
||||
_ = i.search.IndexTorrents(indexedTorrents)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
err = json.NewEncoder(w).Encode(Response{
|
||||
Results: indexedTorrents,
|
||||
Count: len(indexedTorrents),
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getTorrentsTorrentDosFilmes(ctx context.Context, i *Indexer, link string) ([]schema.IndexedTorrent, error) {
|
||||
var indexedTorrents []schema.IndexedTorrent
|
||||
doc, err := getDocument(ctx, i, link)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
article := doc.Find("article")
|
||||
title := strings.Replace(article.Find(".title > h1").Text(), " - Download", "", -1)
|
||||
textContent := article.Find("div.content")
|
||||
date := getPublishedDateFromMeta(doc)
|
||||
magnets := textContent.Find("a[href^=\"magnet\"]")
|
||||
var magnetLinks []string
|
||||
magnets.Each(func(i int, s *goquery.Selection) {
|
||||
magnetLink, _ := s.Attr("href")
|
||||
magnetLinks = append(magnetLinks, magnetLink)
|
||||
})
|
||||
|
||||
var audio []schema.Audio
|
||||
var year string
|
||||
var size []string
|
||||
article.Find("div.content p").Each(func(i int, s *goquery.Selection) {
|
||||
// pattern:
|
||||
// Título Traduzido: Fundação
|
||||
// Título Original: Foundation
|
||||
// IMDb: 7,5
|
||||
// Ano de Lançamento: 2023
|
||||
// Gênero: Ação | Aventura | Ficção
|
||||
// Formato: MKV
|
||||
// Qualidade: WEB-DL
|
||||
// Áudio: Português | Inglês
|
||||
// Idioma: Português | Inglês
|
||||
// Legenda: Português
|
||||
// Tamanho: –
|
||||
// Qualidade de Áudio: 10
|
||||
// Qualidade de Vídeo: 10
|
||||
// Duração: 59 Min.
|
||||
// Servidor: Torrent
|
||||
text := s.Text()
|
||||
|
||||
audio = append(audio, findAudioFromText(text)...)
|
||||
y := findYearFromText(text, title)
|
||||
if y != "" {
|
||||
year = y
|
||||
}
|
||||
size = append(size, findSizesFromText(text)...)
|
||||
})
|
||||
|
||||
// find any link from imdb
|
||||
imdbLink := ""
|
||||
article.Find("div.content a").Each(func(i int, s *goquery.Selection) {
|
||||
link, _ := s.Attr("href")
|
||||
_imdbLink, err := getIMDBLink(link)
|
||||
if err == nil {
|
||||
imdbLink = _imdbLink
|
||||
}
|
||||
})
|
||||
|
||||
size = stableUniq(size)
|
||||
|
||||
var chanIndexedTorrent = make(chan schema.IndexedTorrent)
|
||||
|
||||
// for each magnet link, create a new indexed torrent
|
||||
for it, magnetLink := range magnetLinks {
|
||||
it := it
|
||||
go func(it int, magnetLink string) {
|
||||
magnet, err := magnet.ParseMagnetUri(magnetLink)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
releaseTitle := magnet.DisplayName
|
||||
infoHash := magnet.InfoHash.String()
|
||||
trackers := magnet.Trackers
|
||||
magnetAudio := getAudioFromTitle(releaseTitle, audio)
|
||||
|
||||
peer, seed, err := goscrape.GetLeechsAndSeeds(ctx, i.redis, i.metrics, infoHash, trackers)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
title := processTitle(title, magnetAudio)
|
||||
|
||||
// if the number of sizes is equal to the number of magnets, then assign the size to each indexed torrent in order
|
||||
var mySize string
|
||||
if len(size) == len(magnetLinks) {
|
||||
mySize = size[it]
|
||||
}
|
||||
|
||||
ixt := schema.IndexedTorrent{
|
||||
Title: appendAudioISO639_2Code(releaseTitle, magnetAudio),
|
||||
OriginalTitle: title,
|
||||
Details: link,
|
||||
Year: year,
|
||||
IMDB: imdbLink,
|
||||
Audio: magnetAudio,
|
||||
MagnetLink: magnetLink,
|
||||
Date: date,
|
||||
InfoHash: infoHash,
|
||||
Trackers: trackers,
|
||||
LeechCount: peer,
|
||||
SeedCount: seed,
|
||||
Size: mySize,
|
||||
}
|
||||
chanIndexedTorrent <- ixt
|
||||
}(it, magnetLink)
|
||||
}
|
||||
|
||||
for i := 0; i < len(magnetLinks); i++ {
|
||||
it := <-chanIndexedTorrent
|
||||
indexedTorrents = append(indexedTorrents, it)
|
||||
}
|
||||
|
||||
return indexedTorrents, nil
|
||||
}
|
||||
14
cache/redis.go
vendored
14
cache/redis.go
vendored
@@ -9,13 +9,14 @@ import (
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
var (
|
||||
DefaultExpiration = 24 * time.Hour * 180 // 180 days
|
||||
const (
|
||||
DefaultExpiration = 24 * time.Hour * 7 // 7 days
|
||||
IndexerComandoTorrents = "indexer:comando_torrents"
|
||||
)
|
||||
|
||||
type Redis struct {
|
||||
client *redis.Client
|
||||
client *redis.Client
|
||||
defaultExpiration time.Duration
|
||||
}
|
||||
|
||||
func NewRedis() *Redis {
|
||||
@@ -28,15 +29,20 @@ func NewRedis() *Redis {
|
||||
Addr: fmt.Sprintf("%s:6379", redisHost),
|
||||
Password: "",
|
||||
}),
|
||||
defaultExpiration: DefaultExpiration,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Redis) SetDefaultExpiration(expiration time.Duration) {
|
||||
r.defaultExpiration = expiration
|
||||
}
|
||||
|
||||
func (r *Redis) Get(ctx context.Context, key string) ([]byte, error) {
|
||||
return r.client.Get(ctx, key).Bytes()
|
||||
}
|
||||
|
||||
func (r *Redis) Set(ctx context.Context, key string, value []byte) error {
|
||||
return r.client.Set(ctx, key, value, DefaultExpiration).Err()
|
||||
return r.client.Set(ctx, key, value, r.defaultExpiration).Err()
|
||||
}
|
||||
|
||||
func (r *Redis) SetWithExpiration(ctx context.Context, key string, value []byte, expiration time.Duration) error {
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
version: '3.7'
|
||||
version: '3'
|
||||
|
||||
services:
|
||||
torrent-indexer:
|
||||
image:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: felipemarinho97/torrent-indexer:latest
|
||||
container_name: torrent-indexer
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
@@ -14,6 +11,9 @@ services:
|
||||
- indexer
|
||||
environment:
|
||||
- REDIS_HOST=redis
|
||||
- MEILISEARCH_ADDRESS=http://meilisearch:7700
|
||||
- MEILISEARCH_KEY=my-secret-key
|
||||
- FLARESOLVERR_ADDRESS=http://flaresolverr:8191
|
||||
|
||||
redis:
|
||||
image: redis:alpine
|
||||
@@ -22,5 +22,17 @@ services:
|
||||
networks:
|
||||
- indexer
|
||||
|
||||
# This container is not necessary for the indexer to work,
|
||||
# deploy if you want to use the search feature
|
||||
meilisearch:
|
||||
image: getmeili/meilisearch:latest
|
||||
container_name: meilisearch
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- indexer
|
||||
environment:
|
||||
- MEILI_NO_ANALYTICS=true
|
||||
- MEILI_MASTER_KEY=my-secret-key
|
||||
|
||||
networks:
|
||||
indexer:
|
||||
|
||||
21
go.mod
21
go.mod
@@ -1,14 +1,25 @@
|
||||
module github.com/felipemarinho97/torrent-indexer
|
||||
|
||||
go 1.18
|
||||
go 1.22
|
||||
|
||||
require github.com/redis/go-redis/v9 v9.2.0
|
||||
require github.com/redis/go-redis/v9 v9.5.1
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
golang.org/x/net v0.7.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.0 // indirect
|
||||
github.com/prometheus/common v0.50.0 // indirect
|
||||
github.com/prometheus/procfs v0.13.0 // indirect
|
||||
golang.org/x/sys v0.18.0 // indirect
|
||||
google.golang.org/protobuf v1.33.0 // indirect
|
||||
)
|
||||
|
||||
require github.com/PuerkitoBio/goquery v1.8.1
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.9.1
|
||||
github.com/hbollon/go-edlib v1.6.0
|
||||
github.com/prometheus/client_golang v1.19.0
|
||||
github.com/xhit/go-str2duration/v2 v2.1.0
|
||||
golang.org/x/net v0.22.0
|
||||
)
|
||||
|
||||
49
go.sum
49
go.sum
@@ -1,45 +1,72 @@
|
||||
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
|
||||
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI=
|
||||
github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/redis/go-redis/v9 v9.2.0 h1:zwMdX0A4eVzse46YN18QhuDiM4uf3JmkOB4VZrdt5uI=
|
||||
github.com/redis/go-redis/v9 v9.2.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/hbollon/go-edlib v1.6.0 h1:ga7AwwVIvP8mHm9GsPueC0d71cfRU/52hmPJ7Tprv4E=
|
||||
github.com/hbollon/go-edlib v1.6.0/go.mod h1:wnt6o6EIVEzUfgbUZY7BerzQ2uvzp354qmS2xaLkrhM=
|
||||
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
|
||||
github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
|
||||
github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos=
|
||||
github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8=
|
||||
github.com/prometheus/common v0.50.0 h1:YSZE6aa9+luNa2da6/Tik0q0A5AbR+U003TItK57CPQ=
|
||||
github.com/prometheus/common v0.50.0/go.mod h1:wHFBCEVWVmHMUpg7pYcOm2QUR/ocQdYSJVQJKnHc3xQ=
|
||||
github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o=
|
||||
github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
|
||||
github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
|
||||
github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
|
||||
github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
|
||||
github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
|
||||
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
|
||||
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
|
||||
68
main.go
68
main.go
@@ -1,21 +1,79 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
handler "github.com/felipemarinho97/torrent-indexer/api"
|
||||
"github.com/felipemarinho97/torrent-indexer/cache"
|
||||
"github.com/felipemarinho97/torrent-indexer/monitoring"
|
||||
"github.com/felipemarinho97/torrent-indexer/public"
|
||||
"github.com/felipemarinho97/torrent-indexer/requester"
|
||||
meilisearch "github.com/felipemarinho97/torrent-indexer/search"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
str2duration "github.com/xhit/go-str2duration/v2"
|
||||
)
|
||||
|
||||
func main() {
|
||||
redis := cache.NewRedis()
|
||||
indexers := handler.NewIndexers(redis)
|
||||
searchIndex := meilisearch.NewSearchIndexer(os.Getenv("MEILISEARCH_ADDRESS"), os.Getenv("MEILISEARCH_KEY"), "torrents")
|
||||
metrics := monitoring.NewMetrics()
|
||||
metrics.Register()
|
||||
|
||||
http.HandleFunc("/", handler.HandlerIndex)
|
||||
http.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
|
||||
http.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
|
||||
flaresolverr := requester.NewFlareSolverr(os.Getenv("FLARESOLVERR_ADDRESS"), 60000)
|
||||
req := requester.NewRequester(flaresolverr, redis)
|
||||
|
||||
err := http.ListenAndServe(":7006", nil)
|
||||
// get shot-lived and long-lived cache expiration from env
|
||||
shortLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("SHORT_LIVED_CACHE_EXPIRATION"))
|
||||
if err == nil {
|
||||
fmt.Printf("Setting short-lived cache expiration to %s\n", shortLivedCacheExpiration)
|
||||
req.SetShortLivedCacheExpiration(shortLivedCacheExpiration)
|
||||
}
|
||||
longLivedCacheExpiration, err := str2duration.ParseDuration(os.Getenv("LONG_LIVED_CACHE_EXPIRATION"))
|
||||
if err == nil {
|
||||
fmt.Printf("Setting long-lived cache expiration to %s\n", longLivedCacheExpiration)
|
||||
redis.SetDefaultExpiration(longLivedCacheExpiration)
|
||||
} else {
|
||||
fmt.Println(err)
|
||||
}
|
||||
|
||||
indexers := handler.NewIndexers(redis, metrics, req, searchIndex)
|
||||
search := handler.NewMeilisearchHandler(searchIndex)
|
||||
|
||||
indexerMux := http.NewServeMux()
|
||||
metricsMux := http.NewServeMux()
|
||||
|
||||
indexerMux.HandleFunc("/", handler.HandlerIndex)
|
||||
indexerMux.HandleFunc("/indexers/bludv", indexers.HandlerBluDVIndexer)
|
||||
indexerMux.HandleFunc("/indexers/comando_torrents", indexers.HandlerComandoIndexer)
|
||||
indexerMux.HandleFunc("/indexers/comandohds", indexers.HandlerComandoHDsIndexer)
|
||||
indexerMux.HandleFunc("/indexers/rede_torrent", indexers.HandlerRedeTorrentIndexer)
|
||||
indexerMux.HandleFunc("/indexers/starck-filmes", indexers.HandlerStarckFilmesIndexer)
|
||||
indexerMux.HandleFunc("/indexers/torrent-dos-filmes", indexers.HandlerTorrentDosFilmesIndexer)
|
||||
indexerMux.HandleFunc("/indexers/manual", indexers.HandlerManualIndexer)
|
||||
indexerMux.HandleFunc("/search", search.SearchTorrentHandler)
|
||||
indexerMux.HandleFunc("/search/health", search.HealthHandler)
|
||||
indexerMux.HandleFunc("/search/stats", search.StatsHandler)
|
||||
indexerMux.Handle("/ui/", http.StripPrefix("/ui/", http.FileServer(http.FS(public.UIFiles))))
|
||||
|
||||
metricsMux.Handle("/metrics", promhttp.Handler())
|
||||
|
||||
go func() {
|
||||
err := http.ListenAndServe(":8081", metricsMux)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
port := os.Getenv("PORT")
|
||||
if port == "" {
|
||||
port = "7007"
|
||||
}
|
||||
|
||||
fmt.Printf("Server listening on :%s\n", port)
|
||||
err = http.ListenAndServe(":"+port, indexerMux)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
47
monitoring/prometheus.go
Normal file
47
monitoring/prometheus.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
type Metrics struct {
|
||||
IndexerDuration *prometheus.HistogramVec
|
||||
IndexerErrors *prometheus.CounterVec
|
||||
IndexerRequests *prometheus.CounterVec
|
||||
CacheHits *prometheus.CounterVec
|
||||
CacheMisses *prometheus.CounterVec
|
||||
}
|
||||
|
||||
func NewMetrics() *Metrics {
|
||||
return &Metrics{
|
||||
IndexerDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "indexer_duration_seconds",
|
||||
Help: "Duration of indexer requests",
|
||||
Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 20, 30},
|
||||
}, []string{"indexer"}),
|
||||
IndexerErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "indexer_errors_total",
|
||||
Help: "Number of indexer errors",
|
||||
}, []string{"indexer"}),
|
||||
IndexerRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "indexer_requests_total",
|
||||
Help: "Number of indexer requests",
|
||||
}, []string{"indexer"}),
|
||||
CacheHits: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "cache_hits_total",
|
||||
Help: "Number of cache hits",
|
||||
}, []string{"cache"}),
|
||||
CacheMisses: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "cache_misses_total",
|
||||
Help: "Number of cache misses",
|
||||
}, []string{"cache"}),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) Register() {
|
||||
prometheus.MustRegister(m.IndexerDuration)
|
||||
prometheus.MustRegister(m.IndexerErrors)
|
||||
prometheus.MustRegister(m.IndexerRequests)
|
||||
prometheus.MustRegister(m.CacheHits)
|
||||
prometheus.MustRegister(m.CacheMisses)
|
||||
}
|
||||
6
public/index.go
Normal file
6
public/index.go
Normal file
@@ -0,0 +1,6 @@
|
||||
package public
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed *
|
||||
var UIFiles embed.FS
|
||||
226
public/index.html
Normal file
226
public/index.html
Normal file
@@ -0,0 +1,226 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Torrent Indexer</title>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/@heroicons/react/solid@2.0.0/dist/index.umd.js"></script>
|
||||
</head>
|
||||
|
||||
<body class="bg-gray-900 text-white font-sans min-h-screen flex flex-col">
|
||||
<div class="container mx-auto p-6 flex-grow">
|
||||
<!-- Header -->
|
||||
<header class="text-center mb-10">
|
||||
<h1 class="text-4xl font-bold text-blue-400">Torrent Indexer 🇧🇷</h1>
|
||||
<p class="text-gray-400 mt-2">Find torrents with detailed information from torrent-indexer cache</p>
|
||||
</header>
|
||||
|
||||
<!-- Search Bar -->
|
||||
<div class="flex justify-center mb-10">
|
||||
<input id="search-query" type="text" placeholder="Enter search query"
|
||||
class="w-full max-w-lg px-4 py-2 rounded-md border border-gray-600 bg-gray-800 text-white focus:ring focus:ring-blue-500">
|
||||
<button id="search-btn"
|
||||
class="ml-4 px-6 py-2 bg-blue-600 hover:bg-blue-700 rounded-md font-bold text-white">Search</button>
|
||||
</div>
|
||||
|
||||
<!-- Results Section -->
|
||||
<div id="results" class="space-y-6 mb-10">
|
||||
<!-- Dynamic content will be injected here -->
|
||||
</div>
|
||||
|
||||
<!-- Health Warning -->
|
||||
<div id="health-warning" class="hidden mb-6 p-4 bg-yellow-800 border border-yellow-600 rounded-lg">
|
||||
<div class="flex items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5 text-yellow-400" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 9v2m0 4h.01m-9 4h18a2 2 0 002-2V7a2 2 0 00-2-2H3a2 2 0 00-2 2v10a2 2 0 002 2z" />
|
||||
</svg>
|
||||
<span class="font-bold text-yellow-400">Service Warning</span>
|
||||
</div>
|
||||
<p class="text-yellow-200 mt-2">Search functionality may be disabled or experiencing issues. Please try again later.</p>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- Stats Section -->
|
||||
<div id="database-statistics" class="stats-info mt-auto mb-1 p-3 rounded text-center">
|
||||
<span id="torrentStats" class="text-gray-400">Loading stats...</span>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Global variables
|
||||
let serviceHealthy = true;
|
||||
|
||||
// Function to check service health
|
||||
async function checkHealth() {
|
||||
try {
|
||||
const response = await fetch('/search/health');
|
||||
const health = await response.json();
|
||||
|
||||
if (response.status === 503 || health.status === 'unhealthy') {
|
||||
serviceHealthy = false;
|
||||
showHealthWarning();
|
||||
hideDatabaseStatistics();
|
||||
} else if (health.status === 'degraded') {
|
||||
serviceHealthy = true; // Still operational
|
||||
showHealthWarning(); // But show warning
|
||||
} else {
|
||||
serviceHealthy = true;
|
||||
hideHealthWarning();
|
||||
}
|
||||
} catch (error) {
|
||||
serviceHealthy = false;
|
||||
showHealthWarning();
|
||||
console.error('Health check failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Function to show health warning
|
||||
function showHealthWarning() {
|
||||
document.getElementById('health-warning').classList.remove('hidden');
|
||||
}
|
||||
|
||||
// Function to hide health warning
|
||||
function hideHealthWarning() {
|
||||
document.getElementById('health-warning').classList.add('hidden');
|
||||
}
|
||||
|
||||
function hideDatabaseStatistics() {
|
||||
document.getElementById('database-statistics').classList.add('hidden');
|
||||
}
|
||||
|
||||
// Function to load stats
|
||||
async function loadStats() {
|
||||
try {
|
||||
const response = await fetch('/search/stats');
|
||||
if (response.ok) {
|
||||
const stats = await response.json();
|
||||
const statsElement = document.getElementById('torrentStats');
|
||||
|
||||
const formattedStats = `
|
||||
<span class="text-sm text-gray-500">
|
||||
<span class="text-green-400 font-medium">${stats.numberOfDocuments?.toLocaleString()+'+' || 'N/A'}</span> indexed torrents!
|
||||
</span>
|
||||
`;
|
||||
|
||||
statsElement.innerHTML = formattedStats;
|
||||
} else {
|
||||
throw new Error('Failed to load stats');
|
||||
}
|
||||
} catch (error) {
|
||||
hideDatabaseStatistics();
|
||||
console.error('Stats loading failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Function to render a single torrent result
|
||||
function renderTorrent(torrent) {
|
||||
return `
|
||||
<div class="p-6 bg-gray-800 rounded-lg shadow-md flex flex-col md:flex-row gap-6">
|
||||
<!-- Torrent Title and Details -->
|
||||
<div class="flex-grow">
|
||||
<h2 class="text-2xl font-bold text-blue-400 flex items-center gap-2">
|
||||
<span>${torrent.title}</span>
|
||||
<span class="text-sm text-gray-400">(${torrent.year})</span>
|
||||
</h2>
|
||||
<p class="text-gray-500 italic mt-1">${torrent.original_title}</p>
|
||||
<div class="mt-4 grid grid-cols-1 sm:grid-cols-2 gap-4">
|
||||
<p><strong>Audio:</strong> ${torrent.audio.join(', ')}</p>
|
||||
<p><strong>Size:</strong> ${torrent.size}</p>
|
||||
<p><strong>Seeds:</strong> ${torrent.seed_count} | <strong>Leeches:</strong> ${torrent.leech_count}</p>
|
||||
<p><strong>Info Hash:</strong> <span class="text-sm break-all text-gray-300">${torrent.info_hash}</span></p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Actions -->
|
||||
<div class="flex flex-col justify-between items-start md:items-end">
|
||||
<div>
|
||||
<a href="${torrent.imdb}" target="_blank"
|
||||
class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M8 16l-4-4m0 0l4-4m-4 4h16" />
|
||||
</svg>
|
||||
View on IMDB
|
||||
</a>
|
||||
<a href="${torrent.details}" target="_blank"
|
||||
class="flex items-center gap-2 text-blue-500 hover:text-blue-400 font-medium mt-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M13 16h-1v-4h-.01M9 20h6a2 2 0 002-2v-5a2 2 0 00-2-2h-3.5a2 2 0 00-1.85 1.19M13 10V6a3 3 0 00-6 0v4" />
|
||||
</svg>
|
||||
View Details
|
||||
</a>
|
||||
</div>
|
||||
<a href="${torrent.magnet_link}" target="_blank"
|
||||
class="px-4 py-2 bg-green-600 hover:bg-green-700 text-white font-bold rounded-md flex items-center gap-2 mt-4">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M9 17v-6m6 6v-6m-6 6l-2-2m8 0l2-2M5 9l7-7 7 7" />
|
||||
</svg>
|
||||
Download Magnet
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
// Handle search
|
||||
async function onSearch() {
|
||||
const query = document.getElementById('search-query').value.trim();
|
||||
if (!query) {
|
||||
alert('Please enter a search query!');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`/search?q=${encodeURIComponent(query)}`);
|
||||
if (!response.ok) {
|
||||
throw new Error('Search failed');
|
||||
}
|
||||
|
||||
const results = await response.json();
|
||||
const resultsContainer = document.getElementById('results');
|
||||
|
||||
if (results.length === 0) {
|
||||
resultsContainer.innerHTML = `
|
||||
<div class="p-6 bg-gray-800 rounded-lg shadow-md text-center">
|
||||
<p class="text-xl font-bold text-gray-400">No results found</p>
|
||||
<p class="text-gray-500 mt-2">Try different search terms or check spelling</p>
|
||||
</div>
|
||||
`;
|
||||
} else {
|
||||
resultsContainer.innerHTML = results.map(renderTorrent).join('');
|
||||
}
|
||||
} catch (error) {
|
||||
document.getElementById('results').innerHTML = `
|
||||
<div class="p-6 bg-red-800 rounded-lg shadow-md text-center">
|
||||
<p class="text-xl font-bold text-red-400">Error fetching search results</p>
|
||||
<p class="text-gray-400 mt-2">Please try again later.</p>
|
||||
</div>
|
||||
`;
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
// Event listeners
|
||||
document.getElementById('search-btn').addEventListener('click', onSearch);
|
||||
document.getElementById('search-query').addEventListener('keypress', (e) => {
|
||||
if (e.key === 'Enter') {
|
||||
onSearch();
|
||||
}
|
||||
});
|
||||
|
||||
// Initialize page
|
||||
document.addEventListener('DOMContentLoaded', async () => {
|
||||
await checkHealth();
|
||||
await loadStats();
|
||||
|
||||
// Refresh health and stats periodically
|
||||
setInterval(checkHealth, 30000); // Check health every 30 seconds
|
||||
setInterval(loadStats, 60000); // Update stats every minute
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
300
requester/flaresolverr.go
Normal file
300
requester/flaresolverr.go
Normal file
@@ -0,0 +1,300 @@
|
||||
package requester
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
)
|
||||
|
||||
type FlareSolverr struct {
|
||||
url string
|
||||
maxTimeout int
|
||||
httpClient *http.Client
|
||||
sessionPool chan string
|
||||
mu sync.Mutex
|
||||
initiated bool
|
||||
}
|
||||
|
||||
var (
|
||||
ErrListSessions = fmt.Errorf("failed to list sessions")
|
||||
)
|
||||
|
||||
func NewFlareSolverr(url string, timeoutMilli int) *FlareSolverr {
|
||||
poolSize := 5
|
||||
httpClient := &http.Client{}
|
||||
sessionPool := make(chan string, poolSize) // Pool size of 5 sessions
|
||||
|
||||
f := &FlareSolverr{
|
||||
url: url,
|
||||
maxTimeout: timeoutMilli,
|
||||
httpClient: httpClient,
|
||||
sessionPool: sessionPool,
|
||||
}
|
||||
|
||||
err := f.FillSessionPool()
|
||||
if err == nil {
|
||||
f.initiated = true
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *FlareSolverr) FillSessionPool() error {
|
||||
// Check if the pool is already filled
|
||||
if len(f.sessionPool) == cap(f.sessionPool) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pre-initialize the pool with existing sessions
|
||||
sessions, err := f.ListSessions()
|
||||
if err != nil {
|
||||
// if fail to list sessions, it may not support the sessions.list command
|
||||
// create new dumb sessions to fill the pool
|
||||
if err == ErrListSessions {
|
||||
for len(f.sessionPool) < cap(f.sessionPool) {
|
||||
f.sessionPool <- "dumb-session"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fmt.Println("Failed to list existing FlareSolverr sessions:", err)
|
||||
return err
|
||||
} else {
|
||||
for _, session := range sessions {
|
||||
// Add available sessions to the pool
|
||||
if len(f.sessionPool) < cap(f.sessionPool) {
|
||||
f.sessionPool <- session
|
||||
}
|
||||
}
|
||||
if len(f.sessionPool) > 0 {
|
||||
fmt.Printf("Added %d FlareSolverr sessions to the pool\n", len(f.sessionPool))
|
||||
}
|
||||
}
|
||||
|
||||
// If fewer than poolSize sessions were found, create new ones to fill the pool
|
||||
for len(f.sessionPool) < cap(f.sessionPool) {
|
||||
f.CreateSession()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FlareSolverr) CreateSession() string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
body := map[string]string{"cmd": "sessions.create"}
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := f.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
var sessionResponse map[string]interface{}
|
||||
err = json.NewDecoder(resp.Body).Decode(&sessionResponse)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
session := sessionResponse["session"].(string)
|
||||
// Add session to the pool
|
||||
f.sessionPool <- session
|
||||
|
||||
fmt.Println("Created new FlareSolverr session:", session)
|
||||
return session
|
||||
}
|
||||
|
||||
func (f *FlareSolverr) ListSessions() ([]string, error) {
|
||||
body := map[string]string{"cmd": "sessions.list"}
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := f.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
var sessionsResponse map[string]interface{}
|
||||
err = json.NewDecoder(resp.Body).Decode(&sessionsResponse)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if sessionsResponse["sessions"] == nil {
|
||||
return nil, ErrListSessions
|
||||
}
|
||||
|
||||
sessions := sessionsResponse["sessions"].([]interface{})
|
||||
var sessionIDs []string
|
||||
for _, session := range sessions {
|
||||
sessionIDs = append(sessionIDs, session.(string))
|
||||
}
|
||||
|
||||
return sessionIDs, nil
|
||||
}
|
||||
|
||||
func (f *FlareSolverr) RetrieveSession() string {
|
||||
// Blocking receive from the session pool.
|
||||
session := <-f.sessionPool
|
||||
return session
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
Solution struct {
|
||||
Url string `json:"url"`
|
||||
Status int `json:"status"`
|
||||
Cookies []struct {
|
||||
Domain string `json:"domain"`
|
||||
Expiry int `json:"expiry"`
|
||||
HttpOnly bool `json:"httpOnly"`
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
SameSite string `json:"sameSite"`
|
||||
Secure bool `json:"secure"`
|
||||
Value string `json:"value"`
|
||||
} `json:"cookies"`
|
||||
UserAgent string `json:"userAgent"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
Response string `json:"response"`
|
||||
} `json:"solution"`
|
||||
}
|
||||
|
||||
func (f *FlareSolverr) Get(_url string) (io.ReadCloser, error) {
|
||||
// Check if the FlareSolverr instance was initiated
|
||||
if !f.initiated {
|
||||
return io.NopCloser(bytes.NewReader([]byte(""))), nil
|
||||
}
|
||||
|
||||
// Retrieve session from the pool (blocking if no sessions available)
|
||||
session := f.RetrieveSession()
|
||||
|
||||
// Ensure the session is returned to the pool after the request is done
|
||||
defer func() {
|
||||
f.sessionPool <- session
|
||||
}()
|
||||
|
||||
body := map[string]string{
|
||||
"cmd": "request.get",
|
||||
"url": _url,
|
||||
"maxTimeout": fmt.Sprintf("%d", f.maxTimeout),
|
||||
"session": session,
|
||||
}
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1", f.url), bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := f.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Parse the response
|
||||
var response Response
|
||||
err = json.NewDecoder(resp.Body).Decode(&response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check if the response was successful
|
||||
if response.Status != "ok" {
|
||||
return nil, fmt.Errorf("failed to get response: %s", response.Message)
|
||||
}
|
||||
|
||||
// Check if "Under attack" is in the response
|
||||
if strings.Contains(response.Solution.Response, "Under attack") {
|
||||
return nil, fmt.Errorf("under attack")
|
||||
}
|
||||
|
||||
// check if the response is valid HTML
|
||||
if !utils.IsValidHTML(response.Solution.Response) {
|
||||
fmt.Printf("[FlareSolverr] Invalid HTML response from %s\n", _url)
|
||||
response.Solution.Response = ""
|
||||
}
|
||||
|
||||
// If the response body is empty but cookies are present, make a new request
|
||||
if response.Solution.Response == "" && len(response.Solution.Cookies) > 0 {
|
||||
fmt.Printf("[FlareSolverr] Making a new request to %s with cookies\n", _url)
|
||||
// Create a new request with cookies
|
||||
client := &http.Client{}
|
||||
cookieJar, err := cookiejar.New(&cookiejar.Options{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, cookie := range response.Solution.Cookies {
|
||||
cookieJar.SetCookies(&url.URL{Host: cookie.Domain}, []*http.Cookie{
|
||||
{
|
||||
Name: cookie.Name,
|
||||
Value: cookie.Value,
|
||||
Domain: cookie.Domain,
|
||||
Path: cookie.Path,
|
||||
},
|
||||
})
|
||||
}
|
||||
client.Jar = cookieJar
|
||||
|
||||
secondReq, err := http.NewRequest("GET", _url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// use the same user returned by the FlareSolverr
|
||||
secondReq.Header.Set("User-Agent", response.Solution.UserAgent)
|
||||
|
||||
secondResp, err := client.Do(secondReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
respByte := new(bytes.Buffer)
|
||||
_, err = respByte.ReadFrom(secondResp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Return the body of the second request
|
||||
return io.NopCloser(bytes.NewReader(respByte.Bytes())), nil
|
||||
}
|
||||
|
||||
// Return the original response body
|
||||
return io.NopCloser(bytes.NewReader([]byte(response.Solution.Response))), nil
|
||||
}
|
||||
98
requester/requester.go
Normal file
98
requester/requester.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package requester
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/cache"
|
||||
"github.com/felipemarinho97/torrent-indexer/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
cacheKey = "shortLivedCache"
|
||||
)
|
||||
|
||||
var challangeRegex = regexp.MustCompile(`(?i)(just a moment|cf-chl-bypass|under attack)`)
|
||||
|
||||
type Requster struct {
|
||||
fs *FlareSolverr
|
||||
c *cache.Redis
|
||||
httpClient *http.Client
|
||||
shortLivedCacheExpiration time.Duration
|
||||
}
|
||||
|
||||
func NewRequester(fs *FlareSolverr, c *cache.Redis) *Requster {
|
||||
return &Requster{fs: fs, httpClient: &http.Client{}, c: c, shortLivedCacheExpiration: 30 * time.Minute}
|
||||
}
|
||||
|
||||
func (i *Requster) SetShortLivedCacheExpiration(expiration time.Duration) {
|
||||
i.shortLivedCacheExpiration = expiration
|
||||
}
|
||||
|
||||
func (i *Requster) GetDocument(ctx context.Context, url string) (io.ReadCloser, error) {
|
||||
var body io.ReadCloser
|
||||
|
||||
// try request from short-lived cache
|
||||
key := fmt.Sprintf("%s:%s", cacheKey, url)
|
||||
bodyByte, err := i.c.Get(ctx, key)
|
||||
if err == nil {
|
||||
fmt.Printf("returning from short-lived cache: %s\n", url)
|
||||
body = io.NopCloser(bytes.NewReader(bodyByte))
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// try request with plain client
|
||||
resp, err := i.httpClient.Get(url)
|
||||
if err != nil {
|
||||
// try request with flare solverr
|
||||
body, err = i.fs.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
|
||||
}
|
||||
} else {
|
||||
defer resp.Body.Close()
|
||||
body = resp.Body
|
||||
}
|
||||
|
||||
bodyByte, err = io.ReadAll(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body: %w", err)
|
||||
}
|
||||
if hasChallange(bodyByte) {
|
||||
// try request with flare solverr
|
||||
body, err = i.fs.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to do request for url %s: %w", url, err)
|
||||
}
|
||||
bodyByte, err = io.ReadAll(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body: %w", err)
|
||||
}
|
||||
fmt.Printf("request served from flaresolverr: %s\n", url)
|
||||
} else {
|
||||
fmt.Printf("request served from plain client: %s\n", url)
|
||||
}
|
||||
|
||||
// save response to cache if it's not a challange, body is not empty and is valid HTML
|
||||
if !hasChallange(bodyByte) && len(bodyByte) > 0 && utils.IsValidHTML(string(bodyByte)) {
|
||||
err = i.c.SetWithExpiration(ctx, key, bodyByte, i.shortLivedCacheExpiration)
|
||||
if err != nil {
|
||||
fmt.Printf("failed to save response to cache: %v\n", err)
|
||||
}
|
||||
fmt.Printf("saved to cache: %s\n", url)
|
||||
} else {
|
||||
return nil, fmt.Errorf("response is a challange")
|
||||
}
|
||||
|
||||
return io.NopCloser(bytes.NewReader(bodyByte)), nil
|
||||
}
|
||||
|
||||
// hasChallange checks if the body contains a challange by regex matching
|
||||
func hasChallange(body []byte) bool {
|
||||
return challangeRegex.Match(body)
|
||||
}
|
||||
@@ -1,10 +1,14 @@
|
||||
package schema
|
||||
|
||||
import "strings"
|
||||
|
||||
type Audio string
|
||||
|
||||
const (
|
||||
AudioPortuguese = "Português"
|
||||
AudioPortuguese2 = "Portugues"
|
||||
AudioPortuguese3 = "PT-BR"
|
||||
AudioPortuguese4 = "Dublado"
|
||||
AudioEnglish = "Inglês"
|
||||
AudioEnglish2 = "Ingles"
|
||||
AudioSpanish = "Espanhol"
|
||||
@@ -28,11 +32,18 @@ const (
|
||||
AudioThai = "Tailandês"
|
||||
AudioThai2 = "Tailandes"
|
||||
AudioTurkish = "Turco"
|
||||
AudioHindi = "Hindi"
|
||||
AudioFarsi = "Persa"
|
||||
AudioMalay = "Malaio"
|
||||
AudioDutch = "Holandês"
|
||||
AudioDutch2 = "Holandes"
|
||||
)
|
||||
|
||||
var AudioList = []Audio{
|
||||
AudioPortuguese,
|
||||
AudioPortuguese2,
|
||||
AudioPortuguese3,
|
||||
AudioPortuguese4,
|
||||
AudioEnglish,
|
||||
AudioEnglish2,
|
||||
AudioSpanish,
|
||||
@@ -56,27 +67,36 @@ var AudioList = []Audio{
|
||||
AudioThai,
|
||||
AudioThai2,
|
||||
AudioTurkish,
|
||||
AudioHindi,
|
||||
AudioFarsi,
|
||||
AudioMalay,
|
||||
AudioDutch,
|
||||
AudioDutch2,
|
||||
}
|
||||
|
||||
func (a Audio) String() string {
|
||||
return a.toISO639_2()
|
||||
return a.toTag()
|
||||
}
|
||||
|
||||
func GetAudioFromString(s string) *Audio {
|
||||
for _, a := range AudioList {
|
||||
if string(a) == s {
|
||||
if strings.EqualFold(string(a), s) {
|
||||
return &a
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a Audio) toISO639_2() string {
|
||||
func (a Audio) toTag() string {
|
||||
switch a {
|
||||
case AudioPortuguese:
|
||||
return "por"
|
||||
return "brazilian"
|
||||
case AudioPortuguese2:
|
||||
return "por"
|
||||
return "brazilian"
|
||||
case AudioPortuguese3:
|
||||
return "brazilian"
|
||||
case AudioPortuguese4:
|
||||
return "brazilian"
|
||||
case AudioEnglish:
|
||||
return "eng"
|
||||
case AudioEnglish2:
|
||||
@@ -123,6 +143,16 @@ func (a Audio) toISO639_2() string {
|
||||
return "tha"
|
||||
case AudioTurkish:
|
||||
return "tur"
|
||||
case AudioHindi:
|
||||
return "hin"
|
||||
case AudioFarsi:
|
||||
return "fas"
|
||||
case AudioMalay:
|
||||
return "msa"
|
||||
case AudioDutch:
|
||||
return "nld"
|
||||
case AudioDutch2:
|
||||
return "nld"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
|
||||
20
schema/indexed_torrent.go
Normal file
20
schema/indexed_torrent.go
Normal file
@@ -0,0 +1,20 @@
|
||||
package schema
|
||||
|
||||
import "time"
|
||||
|
||||
type IndexedTorrent struct {
|
||||
Title string `json:"title"`
|
||||
OriginalTitle string `json:"original_title"`
|
||||
Details string `json:"details"`
|
||||
Year string `json:"year"`
|
||||
IMDB string `json:"imdb"`
|
||||
Audio []Audio `json:"audio"`
|
||||
MagnetLink string `json:"magnet_link"`
|
||||
Date time.Time `json:"date"`
|
||||
InfoHash string `json:"info_hash"`
|
||||
Trackers []string `json:"trackers"`
|
||||
Size string `json:"size"`
|
||||
LeechCount int `json:"leech_count"`
|
||||
SeedCount int `json:"seed_count"`
|
||||
Similarity float32 `json:"similarity"`
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/cache"
|
||||
"github.com/felipemarinho97/torrent-indexer/monitoring"
|
||||
)
|
||||
|
||||
type peers struct {
|
||||
@@ -44,12 +45,14 @@ func setPeersToCache(ctx context.Context, r *cache.Redis, infoHash string, peer,
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, infoHash string, trackers []string) (int, int, error) {
|
||||
func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, m *monitoring.Metrics, infoHash string, trackers []string) (int, int, error) {
|
||||
leech, seed, err := getPeersFromCache(ctx, r, infoHash)
|
||||
if err != nil {
|
||||
m.CacheHits.WithLabelValues("peers").Inc()
|
||||
fmt.Println("unable to get peers from cache for infohash:", infoHash)
|
||||
} else {
|
||||
fmt.Println("get from cache> leech:", leech, "seed:", seed)
|
||||
m.CacheMisses.WithLabelValues("peers").Inc()
|
||||
fmt.Println("hash:", infoHash, "get from cache -> leech:", leech, "seed:", seed)
|
||||
return leech, seed, nil
|
||||
}
|
||||
|
||||
@@ -84,13 +87,18 @@ func GetLeechsAndSeeds(ctx context.Context, r *cache.Redis, infoHash string, tra
|
||||
var peer peers
|
||||
for i := 0; i < len(trackers); i++ {
|
||||
select {
|
||||
case <-errChan:
|
||||
// discard error
|
||||
case peer = <-peerChan:
|
||||
setPeersToCache(ctx, r, infoHash, peer.Leechers, peer.Seeders)
|
||||
err = setPeersToCache(ctx, r, infoHash, peer.Leechers, peer.Seeders)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
} else {
|
||||
fmt.Println("hash:", infoHash, "get from tracker -> leech:", peer.Leechers, "seed:", peer.Seeders)
|
||||
}
|
||||
return peer.Leechers, peer.Seeders, nil
|
||||
case err := <-errChan:
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
return 0, 0, fmt.Errorf("unable to get peers from trackers")
|
||||
return 0, 0, fmt.Errorf("unable to get peers from trackers for infohash: %s", infoHash)
|
||||
}
|
||||
|
||||
223
search/meilisearch.go
Normal file
223
search/meilisearch.go
Normal file
@@ -0,0 +1,223 @@
|
||||
package meilisearch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/felipemarinho97/torrent-indexer/schema"
|
||||
)
|
||||
|
||||
// SearchIndexer integrates with Meilisearch to index and search torrent items.
|
||||
type SearchIndexer struct {
|
||||
Client *http.Client
|
||||
BaseURL string
|
||||
APIKey string
|
||||
IndexName string
|
||||
}
|
||||
|
||||
// IndexStats represents statistics about the Meilisearch index
|
||||
type IndexStats struct {
|
||||
NumberOfDocuments int64 `json:"numberOfDocuments"`
|
||||
IsIndexing bool `json:"isIndexing"`
|
||||
FieldDistribution map[string]int64 `json:"fieldDistribution"`
|
||||
}
|
||||
|
||||
// HealthStatus represents the health status of Meilisearch
|
||||
type HealthStatus struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
// NewSearchIndexer creates a new instance of SearchIndexer.
|
||||
func NewSearchIndexer(baseURL, apiKey, indexName string) *SearchIndexer {
|
||||
return &SearchIndexer{
|
||||
Client: &http.Client{Timeout: 10 * time.Second},
|
||||
BaseURL: baseURL,
|
||||
APIKey: apiKey,
|
||||
IndexName: indexName,
|
||||
}
|
||||
}
|
||||
|
||||
// IndexTorrent indexes a single torrent item in Meilisearch.
|
||||
func (t *SearchIndexer) IndexTorrent(torrent schema.IndexedTorrent) error {
|
||||
url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
|
||||
|
||||
torrentWithKey := struct {
|
||||
Hash string `json:"id"`
|
||||
schema.IndexedTorrent
|
||||
}{
|
||||
Hash: torrent.InfoHash,
|
||||
IndexedTorrent: torrent,
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(torrentWithKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal torrent data: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if t.APIKey != "" {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
|
||||
}
|
||||
|
||||
resp, err := t.Client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *SearchIndexer) IndexTorrents(torrents []schema.IndexedTorrent) error {
|
||||
url := fmt.Sprintf("%s/indexes/%s/documents", t.BaseURL, t.IndexName)
|
||||
|
||||
torrentsWithKey := make([]struct {
|
||||
Hash string `json:"id"`
|
||||
schema.IndexedTorrent
|
||||
}, 0, len(torrents))
|
||||
for _, torrent := range torrents {
|
||||
torrentWithKey := struct {
|
||||
Hash string `json:"id"`
|
||||
schema.IndexedTorrent
|
||||
}{
|
||||
Hash: torrent.InfoHash,
|
||||
IndexedTorrent: torrent,
|
||||
}
|
||||
torrentsWithKey = append(torrentsWithKey, torrentWithKey)
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(torrentsWithKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal torrent data: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if t.APIKey != "" {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
|
||||
}
|
||||
|
||||
resp, err := t.Client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SearchTorrent searches indexed torrents in Meilisearch based on the query.
|
||||
func (t *SearchIndexer) SearchTorrent(query string, limit int) ([]schema.IndexedTorrent, error) {
|
||||
url := fmt.Sprintf("%s/indexes/%s/search", t.BaseURL, t.IndexName)
|
||||
requestBody := map[string]string{
|
||||
"q": query,
|
||||
}
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal search query: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if t.APIKey != "" {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
|
||||
}
|
||||
|
||||
resp, err := t.Client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to execute request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("search failed: %s", body)
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Hits []schema.IndexedTorrent `json:"hits"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse search response: %w", err)
|
||||
}
|
||||
|
||||
return result.Hits, nil
|
||||
}
|
||||
|
||||
// GetStats retrieves statistics about the Meilisearch index including document count.
|
||||
// This method can be used for health checks and monitoring.
|
||||
func (t *SearchIndexer) GetStats() (*IndexStats, error) {
|
||||
url := fmt.Sprintf("%s/indexes/%s/stats", t.BaseURL, t.IndexName)
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
if t.APIKey != "" {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", t.APIKey))
|
||||
}
|
||||
|
||||
resp, err := t.Client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to execute request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("failed to get stats: status %d, body: %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
var stats IndexStats
|
||||
if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse stats response: %w", err)
|
||||
}
|
||||
|
||||
return &stats, nil
|
||||
}
|
||||
|
||||
// IsHealthy checks if Meilisearch is available and responsive.
|
||||
// Returns true if the service is healthy, false otherwise.
|
||||
func (t *SearchIndexer) IsHealthy() bool {
|
||||
url := fmt.Sprintf("%s/health", t.BaseURL)
|
||||
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Use a shorter timeout for health checks
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return resp.StatusCode == http.StatusOK
|
||||
}
|
||||
|
||||
// GetDocumentCount returns the number of indexed documents.
|
||||
// This is a convenience method that extracts just the document count from stats.
|
||||
func (t *SearchIndexer) GetDocumentCount() (int64, error) {
|
||||
stats, err := t.GetStats()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return stats.NumberOfDocuments, nil
|
||||
}
|
||||
28
utils/decoder.go
Normal file
28
utils/decoder.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"html"
|
||||
)
|
||||
|
||||
func DecodeAdLink(encodedStr string) (string, error) {
|
||||
reversed := reverseString(encodedStr)
|
||||
|
||||
decodedBytes, err := base64.StdEncoding.DecodeString(reversed)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
htmlUnescaped := html.UnescapeString(string(decodedBytes))
|
||||
|
||||
return htmlUnescaped, nil
|
||||
}
|
||||
|
||||
// Helper function to reverse a string
|
||||
func reverseString(s string) string {
|
||||
runes := []rune(s)
|
||||
for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
|
||||
runes[i], runes[j] = runes[j], runes[i]
|
||||
}
|
||||
return string(runes)
|
||||
}
|
||||
23
utils/util.go
Normal file
23
utils/util.go
Normal file
@@ -0,0 +1,23 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func Filter[A any](arr []A, f func(A) bool) []A {
|
||||
var res []A
|
||||
res = make([]A, 0)
|
||||
for _, v := range arr {
|
||||
if f(v) {
|
||||
res = append(res, v)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func IsValidHTML(input string) bool {
|
||||
r := strings.NewReader(input)
|
||||
_, err := html.Parse(r)
|
||||
return err == nil
|
||||
}
|
||||
Reference in New Issue
Block a user