From a018209ec022edc84d2ebd81c2331ab62eb418ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20Delattre?= Date: Mon, 9 Mar 2026 11:11:27 +0100 Subject: [PATCH 1/3] Update go and use github actions --- .github/workflows/ci.yml | 16 ++++++++++++++++ .travis.yml | 8 -------- README.md | 3 +-- go.mod | 2 +- 4 files changed, 18 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4d0f4d3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,16 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - run: go test -v . diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1026702..0000000 --- a/.travis.yml +++ /dev/null @@ -1,8 +0,0 @@ -language: go -go: - - 1.13.7 -before_install: - - go get github.com/mattn/goveralls -script: - - go test -v -covermode=count -coverprofile=coverage.out . - - goveralls -service=travis-ci diff --git a/README.md b/README.md index 1dac260..a86521a 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ # Golang scrapper for thepiratebay -[![Build Status](https://travis-ci.org/odwrtw/tpb.svg?branch=master)](https://travis-ci.org/odwrtw/tpb) +[![CI](https://github.com/odwrtw/tpb/actions/workflows/ci.yml/badge.svg)](https://github.com/odwrtw/tpb/actions/workflows/ci.yml) [![Go Report Card](https://goreportcard.com/badge/github.com/odwrtw/tpb)](https://goreportcard.com/report/github.com/odwrtw/tpb) [![GoDoc](https://godoc.org/github.com/odwrtw/tpb?status.png)](http://godoc.org/github.com/odwrtw/tpb) -[![Coverage Status](https://coveralls.io/repos/github/odwrtw/tpb/badge.svg?branch=master)](https://coveralls.io/github/odwrtw/tpb?branch=master) ## Exemple diff --git a/go.mod b/go.mod index 281951d..3fb474a 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/odwrtw/tpb -go 1.13 +go 1.26 From 1a8cefc7cf82c8a30d29e56c87157a1e48b97005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20Delattre?= Date: Mon, 9 Mar 2026 11:35:15 +0100 Subject: [PATCH 2/3] Update trackers and API calls --- fetch.go | 42 ++++++++++++++--------------------- file.go | 30 +++++++++++++++++++++++++ magnet.go | 24 ++++++++++---------- magnet_test.go | 14 +++++++++++- torrent.go | 58 +++++++++++++++++++++++++++---------------------- torrent_test.go | 47 +++++++++++++++++++++++++++++++++------ tpb.go | 16 +++++++++----- 7 files changed, 155 insertions(+), 76 deletions(-) create mode 100644 file.go diff --git a/fetch.go b/fetch.go index 50e38c0..6b072d3 100644 --- a/fetch.go +++ b/fetch.go @@ -25,7 +25,7 @@ func (c *Client) fetchTorrents(ctx context.Context, path string) ([]*Torrent, er // fetch will try to GET the path, trying all the endpoints if needed, and // unmarshal the results in the data interface -func (c *Client) fetch(ctx context.Context, path string, data interface{}) error { +func (c *Client) fetch(ctx context.Context, path string, data any) error { var err error for i := 0; i < c.MaxTries; i++ { endpoint := c.endpoints.best() @@ -36,7 +36,7 @@ func (c *Client) fetch(ctx context.Context, path string, data interface{}) error timeoutCtx, cancel := context.WithTimeout(ctx, c.EndpointTimeout) defer cancel() - err = get(timeoutCtx, endpoint.baseURL+path, &data) + err = get(timeoutCtx, endpoint.baseURL+path, data) if err == nil { return nil } @@ -53,31 +53,21 @@ func (c *Client) fetch(ctx context.Context, path string, data interface{}) error } // get will GET the url and unmarshal the results in the data interface -func get(ctx context.Context, url string, data interface{}) error { - var err error - done := make(chan struct{}) - - go func() { - defer close(done) - var resp *http.Response - resp, err = http.Get(url) - if err != nil { - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - err = fmt.Errorf("got status %d when making the request", resp.StatusCode) - return - } - - err = json.NewDecoder(resp.Body).Decode(&data) - }() +func get(ctx context.Context, url string, data any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } - select { - case <-done: + resp, err := http.DefaultClient.Do(req) + if err != nil { return err - case <-ctx.Done(): - return ctx.Err() } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("got status %d when making the request", resp.StatusCode) + } + + return json.NewDecoder(resp.Body).Decode(data) } diff --git a/file.go b/file.go new file mode 100644 index 0000000..c16e68a --- /dev/null +++ b/file.go @@ -0,0 +1,30 @@ +package tpb + +import ( + "encoding/json" + "path" +) + +// File represents a file within a torrent +type File struct { + // Name is the full path of the file, joined from the path components + // returned by the API + Name string + // Size is the file size in bytes + Size int64 +} + +// UnmarshalJSON handles the API's format where name is an array of path +// components and size is a string +func (f *File) UnmarshalJSON(data []byte) error { + var aux struct { + Name []string `json:"name"` + Size flexInt `json:"size"` + } + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + f.Name = path.Join(aux.Name...) + f.Size = int64(aux.Size) + return nil +} diff --git a/magnet.go b/magnet.go index f8d92c2..ed452b9 100644 --- a/magnet.go +++ b/magnet.go @@ -14,18 +14,21 @@ func init() { const magnetTemplateText = `magnet:?xt=urn:btih:{{.InfoHash}}&dn={{.Name}}{{range .Trackers}}&tr={{.}}{{end}}` var trackers = []string{ - "udp://tracker.coppersurfer.tk:6969/announce", - "udp://9.rarbg.to:2920/announce", - "udp://tracker.opentrackr.org:1337", - "udp://tracker.internetwarriors.net:1337/announce", - "udp://tracker.leechers-paradise.org:6969/announce", - "udp://tracker.coppersurfer.tk:6969/announce", - "udp://tracker.pirateparty.gr:6969/announce", - "udp://tracker.cyberia.is:6969/announce", + "udp://tracker.opentrackr.org:1337/announce", + "udp://open.tracker.cl:1337/announce", + "udp://tracker.openbittorrent.com:6969/announce", + "udp://opentracker.i2p.rocks:6969/announce", + "udp://tracker.torrent.eu.org:451/announce", + "udp://open.stealth.si:80/announce", } -// Magnet returns a Magnet for a torrent +// Magnet returns a magnet URI for the torrent. If the API provided a magnet +// link directly, it is returned as-is. Otherwise one is constructed from the +// info hash and a set of well-known trackers. func (t *Torrent) Magnet() string { + if t.MagnetLink != "" { + return t.MagnetLink + } var tpl bytes.Buffer tplStruct := struct { *Torrent @@ -34,8 +37,7 @@ func (t *Torrent) Magnet() string { Torrent: t, Trackers: trackers, } - err := magnetTemplate.Execute(&tpl, tplStruct) - if err != nil { + if err := magnetTemplate.Execute(&tpl, tplStruct); err != nil { return "" } return tpl.String() diff --git a/magnet_test.go b/magnet_test.go index c928338..40a8ebe 100644 --- a/magnet_test.go +++ b/magnet_test.go @@ -21,9 +21,21 @@ func TestMagnet(t *testing.T) { Description: "description of Big Buck Bunny", Added: time.Unix(1509051120, 0), } - expectedMagnet := `magnet:?xt=urn:btih:363BC69191230430C6758318D196CCD61DB61B647&dn=Big Buck Bunny&tr=udp://tracker.coppersurfer.tk:6969/announce&tr=udp://9.rarbg.to:2920/announce&tr=udp://tracker.opentrackr.org:1337&tr=udp://tracker.internetwarriors.net:1337/announce&tr=udp://tracker.leechers-paradise.org:6969/announce&tr=udp://tracker.coppersurfer.tk:6969/announce&tr=udp://tracker.pirateparty.gr:6969/announce&tr=udp://tracker.cyberia.is:6969/announce` + expectedMagnet := `magnet:?xt=urn:btih:363BC69191230430C6758318D196CCD61DB61B647&dn=Big Buck Bunny&tr=udp://tracker.opentrackr.org:1337/announce&tr=udp://open.tracker.cl:1337/announce&tr=udp://tracker.openbittorrent.com:6969/announce&tr=udp://opentracker.i2p.rocks:6969/announce&tr=udp://tracker.torrent.eu.org:451/announce&tr=udp://open.stealth.si:80/announce` magnet := torrent.Magnet() if magnet != expectedMagnet { t.Fatalf("expected magnet %q, got %q", expectedMagnet, magnet) } } + +func TestMagnetFromAPI(t *testing.T) { + apiMagnet := "magnet:?xt=urn:btih:AABBCC&dn=Some+Torrent&tr=udp://tracker.example.com:6969/announce" + torrent := &Torrent{ + InfoHash: "AABBCC", + Name: "Some Torrent", + MagnetLink: apiMagnet, + } + if got := torrent.Magnet(); got != apiMagnet { + t.Fatalf("expected API magnet %q, got %q", apiMagnet, got) + } +} diff --git a/torrent.go b/torrent.go index df6bc15..b0e655d 100644 --- a/torrent.go +++ b/torrent.go @@ -9,38 +9,42 @@ import ( // Torrent represents a Torrent type Torrent struct { - ID int `json:"id"` - Category TorrentCategory `json:"category"` - Status UserStatus `json:"status"` - Name string `json:"name"` - NumFiles int `json:"num_files"` - Size uint64 `json:"size"` - Seeders int `json:"seeders"` - Leechers int `json:"leechers"` - User string `json:"username"` - Added time.Time `json:"added"` - Description string `json:"descripiton"` - InfoHash string `json:"info_hash"` - ImdbID string `json:"imdb"` + ID int `json:"id"` + Category TorrentCategory `json:"category"` + CategoryName string `json:"category_name"` + Status UserStatus `json:"status"` + Name string `json:"name"` + NumFiles int `json:"num_files"` + Size uint64 `json:"size"` + Seeders int `json:"seeders"` + Leechers int `json:"leechers"` + User string `json:"username"` + Added time.Time `json:"added"` + Description string `json:"description"` + InfoHash string `json:"info_hash"` + ImdbID string `json:"imdb"` + MagnetLink string `json:"magnet_link"` } // UnmarshalJSON is a custom unmarshal function to handle timestamps and // boolean as int and convert them to the right type. func (t *Torrent) UnmarshalJSON(data []byte) error { var aux struct { - ID flexInt `json:"id"` - Category flexInt `json:"category"` - Status string `json:"status"` - Name string `json:"name"` - NumFiles flexInt `json:"num_files"` - InfoHash string `json:"info_hash"` - Description string `json:"descr"` - Leechers flexInt `json:"leechers"` - Seeders flexInt `json:"seeders"` - User string `json:"username"` - Size flexInt `json:"size"` - Added flexInt `json:"added"` - ImdbID flexString `json:"imdb"` + ID flexInt `json:"id"` + Category flexInt `json:"category"` + CategoryName string `json:"category_name"` + Status string `json:"status"` + Name string `json:"name"` + NumFiles flexInt `json:"num_files"` + InfoHash string `json:"info_hash"` + Description string `json:"descr"` + Leechers flexInt `json:"leechers"` + Seeders flexInt `json:"seeders"` + User string `json:"username"` + Size flexInt `json:"size"` + Added flexInt `json:"added"` + ImdbID flexString `json:"imdb"` + MagnetLink string `json:"magnet_link"` } // Decode json into the aux struct @@ -50,6 +54,7 @@ func (t *Torrent) UnmarshalJSON(data []byte) error { t.ID = int(aux.ID) t.Category = TorrentCategory(int(aux.Category)) + t.CategoryName = aux.CategoryName t.Status = UserStatus(aux.Status) t.Name = aux.Name t.NumFiles = int(aux.NumFiles) @@ -61,6 +66,7 @@ func (t *Torrent) UnmarshalJSON(data []byte) error { t.Description = aux.Description t.InfoHash = aux.InfoHash t.ImdbID = string(aux.ImdbID) + t.MagnetLink = aux.MagnetLink return nil } diff --git a/torrent_test.go b/torrent_test.go index 6c217cc..11189fa 100644 --- a/torrent_test.go +++ b/torrent_test.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" "net/http/httptest" - "net/url" "reflect" "testing" "time" @@ -85,7 +84,7 @@ func TestSearch(t *testing.T) { defer ts.Close() expected := []*Torrent{ - &Torrent{ + { ID: 6665688, Name: "Big Buck Bunny", InfoHash: "363BC69191230430C6758318D196CCD61DB61B647", @@ -127,7 +126,7 @@ func TestUser(t *testing.T) { defer ts.Close() expected := []*Torrent{ - &Torrent{ + { ID: 18837600, Name: "ArchLinux", InfoHash: "B137DE1DF926E787FE263D4187B34B23", @@ -154,7 +153,7 @@ func TestUser(t *testing.T) { t.Fatalf("expected: \n%+v\n, got \n%+v", expected, got) } - expectedRequestURI := "/q.php?q=" + url.QueryEscape("user:user1:0") + expectedRequestURI := "/u.php?page=0&u=user1" if requestURI != expectedRequestURI { t.Fatalf("expected URL %q, got %q", expectedRequestURI, requestURI) } @@ -169,7 +168,7 @@ func TestCategory(t *testing.T) { defer ts.Close() expected := []*Torrent{ - &Torrent{ + { ID: 6665688, Name: "Big Buck Bunny", InfoHash: "363BC69191230430C6758318D196CCD61DB61B647", @@ -196,7 +195,7 @@ func TestCategory(t *testing.T) { t.Fatalf("expected: \n%+v\n, got \n%+v", expected, got) } - expectedRequestURI := "/q.php?q=" + url.QueryEscape("category:0:0") + expectedRequestURI := "/q.php?q=category%3A0%3A0" if requestURI != expectedRequestURI { t.Fatalf("expected URL %q, got %q", expectedRequestURI, requestURI) } @@ -211,7 +210,7 @@ func TestTop100(t *testing.T) { defer ts.Close() expected := []*Torrent{ - &Torrent{ + { ID: 6665688, Name: "Big Buck Bunny", InfoHash: "363BC69191230430C6758318D196CCD61DB61B647", @@ -284,3 +283,37 @@ func TestTorrentInfo(t *testing.T) { t.Fatalf("expected URL %q, got %q", expectedRequestURI, requestURI) } } + +var rawFileList = `[ + {"name": ["Big Buck Bunny", "BigBuckBunny.mp4"], "size": "736780288"}, + {"name": ["Big Buck Bunny", "poster.jpg"], "size": "1314718"} +]` + +func TestFileList(t *testing.T) { + var requestURI string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestURI = r.RequestURI + fmt.Fprint(w, rawFileList) + })) + defer ts.Close() + + expected := []*File{ + {Name: "Big Buck Bunny/BigBuckBunny.mp4", Size: 736780288}, + {Name: "Big Buck Bunny/poster.jpg", Size: 1314718}, + } + + client := New(ts.URL) + got, err := client.FileList(context.Background(), 6665688) + if err != nil { + t.Fatalf("got error from FileList: %q", err) + } + + if !reflect.DeepEqual(got, expected) { + t.Fatalf("expected: \n%+v\n, got \n%+v", expected, got) + } + + expectedRequestURI := "/f.php?id=6665688" + if requestURI != expectedRequestURI { + t.Fatalf("expected URL %q, got %q", expectedRequestURI, requestURI) + } +} diff --git a/tpb.go b/tpb.go index be3d160..2fdba27 100644 --- a/tpb.go +++ b/tpb.go @@ -53,10 +53,10 @@ func (c *Client) User(ctx context.Context, user string, opts *UserOptions) ([]*T Page: 0, } } - query := fmt.Sprintf("user:%s:%d", user, opts.Page) v := url.Values{} - v.Add("q", query) - path := "/q.php?" + v.Encode() + v.Set("u", user) + v.Set("page", strconv.Itoa(opts.Page)) + path := "/u.php?" + v.Encode() return c.fetchTorrents(ctx, path) } @@ -93,5 +93,11 @@ func (c *Client) Infos(ctx context.Context, id int) (*Torrent, error) { return t, c.fetch(ctx, path, &t) } -// TODO: Implement FileList -// curl https://apibay.org/f.php?id=36120091 +// FileList returns the list of files in a torrent +func (c *Client) FileList(ctx context.Context, id int) ([]*File, error) { + var files []*File + v := url.Values{} + v.Add("id", strconv.Itoa(id)) + path := "/f.php?" + v.Encode() + return files, c.fetch(ctx, path, &files) +} From f7d50ac0f61fa32e47848559e1b34f4d56f28c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20Delattre?= Date: Mon, 9 Mar 2026 14:26:20 +0100 Subject: [PATCH 3/3] Catch the HTTP status 429 as a rate limit error --- fetch.go | 3 +++ tpb.go | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/fetch.go b/fetch.go index 6b072d3..efca985 100644 --- a/fetch.go +++ b/fetch.go @@ -65,6 +65,9 @@ func get(ctx context.Context, url string, data any) error { } defer resp.Body.Close() + if resp.StatusCode == http.StatusTooManyRequests { + return ErrRateLimited + } if resp.StatusCode != http.StatusOK { return fmt.Errorf("got status %d when making the request", resp.StatusCode) } diff --git a/tpb.go b/tpb.go index 2fdba27..3469742 100644 --- a/tpb.go +++ b/tpb.go @@ -11,8 +11,12 @@ import ( var defaultTimeout = 10 * time.Second -// ErrMissingEndpoint is the error returned if there's no endpoint -var ErrMissingEndpoint = errors.New("tpb: missing endpoint") +var ( + // ErrMissingEndpoint is the error returned if there's no endpoint + ErrMissingEndpoint = errors.New("tpb: missing endpoint") + // ErrRateLimited is returned when the server responds with HTTP 429 + ErrRateLimited = errors.New("tpb: rate limited") +) // Client represent a Client used to make Search type Client struct {