ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: refactor offline download (#5408 close #4108)

Browse Source 
* wip: refactor offline download (#5331)

* base tool

* working: aria2

* refactor: change type of percentage to float64

* wip: adapt aria2

* wip: use items in offline_download

* wip: use tool manager

* wip: adapt qBittorrent

* chore: fix typo

* Squashed commit of the following:

commit 4fc0a77565
Author: Andy Hsu <i@nn.ci>
Date:   Fri Oct 20 21:06:25 2023 +0800

    fix(baidu_netdisk): upload file > 4GB (close #5392)

commit aaffaee2b5
Author: gmugu <94156510@qq.com>
Date:   Thu Oct 19 19:17:53 2023 +0800

    perf(webdav): support request with cookies (#5391)

commit 8ef8023c20
Author: NewbieOrange <NewbieOrange@users.noreply.github.com>
Date:   Thu Oct 19 19:17:09 2023 +0800

    fix(aliyundrive_open): upload progress for normal upload (#5398)

commit cdfbe6dcf2
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Wed Oct 18 16:27:07 2023 +0800

    fix: hash gcid empty file (#5394)

commit 94d028743a
Author: Andy Hsu <i@nn.ci>
Date:   Sat Oct 14 13:17:51 2023 +0800

    ci: remove `pr-welcome` label when close issue [skip ci]

commit 7f7335435c
Author: itsHenry <2671230065@qq.com>
Date:   Sat Oct 14 13:12:46 2023 +0800

    feat(cloudreve): support thumbnail (#5373 close #5348)

    * feat(cloudreve): support thumbnail

    * chore: remove unnecessary code

commit b9e192b29c
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Thu Oct 12 20:57:12 2023 +0800

    fix(115): limit request rate (#5367 close #5275)

    * fix(115):limit request rate

    * chore(115): fix unit of `limit_rate`

    ---------

    Co-authored-by: Andy Hsu <i@nn.ci>

commit 69a98eaef6
Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Date:   Wed Oct 11 22:01:55 2023 +0800

    fix(deps): update module github.com/aliyun/aliyun-oss-go-sdk to v2.2.9+incompatible (#5141)

    Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

commit 1ebc96a4e5
Author: Andy Hsu <i@nn.ci>
Date:   Tue Oct 10 18:32:00 2023 +0800

    fix(wopan): fatal error concurrent map writes (close #5352)

commit 66e2324cac
Author: Andy Hsu <i@nn.ci>
Date:   Tue Oct 10 18:23:11 2023 +0800

    chore(deps): upgrade dependencies

commit 7600dc28df
Author: Andy Hsu <i@nn.ci>
Date:   Tue Oct 10 18:13:58 2023 +0800

    fix(aliyundrive_open): change default api to raw server (close #5358)

commit 8ef89ad0a4
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Tue Oct 10 18:08:27 2023 +0800

    fix(baidu_netdisk): hash and `error 2` (#5356)

    * fix(baidu):hash and error:2

    * fix:invalid memory address

commit 35d672217d
Author: jeffmingup <1960588251@qq.com>
Date:   Sun Oct 8 19:29:45 2023 +0800

    fix(onedrive_app): incorrect api on `_accessToken` (#5346)

commit 1a283bb272
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Fri Oct 6 16:04:39 2023 +0800

    feat(google_drive): add `hash_info`, `ctime`, `thumbnail` (#5334)

commit a008f54f4d
Author: nkh0472 <67589323+nkh0472@users.noreply.github.com>
Date:   Thu Oct 5 13:10:51 2023 +0800

    docs: minor language improvements (#5329) [skip ci]

* fix: adapt update progress type

* Squashed commit of the following:

commit 65c5ec0c34
Author: itsHenry <2671230065@qq.com>
Date:   Sat Nov 4 13:35:09 2023 +0800

    feat(cloudreve): folder size count and switch (#5457 close #5395)

commit a6325967d0
Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Date:   Mon Oct 30 15:11:20 2023 +0800

    fix(deps): update module github.com/charmbracelet/lipgloss to v0.9.1 (#5234)

    Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

commit 4dff49470a
Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Date:   Mon Oct 30 15:10:36 2023 +0800

    fix(deps): update golang.org/x/exp digest to 7918f67 (#5366)

    Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

commit cc86d6f3d1
Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Date:   Sun Oct 29 14:45:55 2023 +0800

    fix(deps): update module golang.org/x/net to v0.17.0 [security] (#5370)

    Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

commit c0f9c8ebaf
Author: Andy Hsu <i@nn.ci>
Date:   Thu Oct 26 19:21:09 2023 +0800

    feat: add ignore direct link params (close #5434)
This commit is contained in:
Andy Hsu
2023-11-06 16:56:55 +08:00
committed by GitHub
parent 3bbdd4fa89
commit 769281bd40
48 changed files with 1238 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

366
pkg/qbittorrent/client.go Normal file
View File

@ -0,0 +1,366 @@
package qbittorrent
import (
"bytes"
"errors"
"io"
"mime/multipart"
"net/http"
"net/http/cookiejar"
"net/url"
"github.com/alist-org/alist/v3/pkg/utils"
)
type Client interface {
AddFromLink(link string, savePath string, id string) error
GetInfo(id string) (TorrentInfo, error)
GetFiles(id string) ([]FileInfo, error)
Delete(id string, deleteFiles bool) error
}
type client struct {
url *url.URL
client http.Client
Client
}
func New(webuiUrl string) (Client, error) {
u, err := url.Parse(webuiUrl)
if err != nil {
return nil, err
}
jar, err := cookiejar.New(nil)
if err != nil {
return nil, err
}
var c = &client{
url: u,
client: http.Client{Jar: jar},
}
err = c.checkAuthorization()
if err != nil {
return nil, err
}
return c, nil
}
func (c *client) checkAuthorization() error {
// check authorization
if c.authorized() {
return nil
}
// check authorization after logging in
err := c.login()
if err != nil {
return err
}
if c.authorized() {
return nil
}
return errors.New("unauthorized qbittorrent url")
}
func (c *client) authorized() bool {
resp, err := c.post("/api/v2/app/version", nil)
if err != nil {
return false
}
return resp.StatusCode == 200 // the status code will be 403 if not authorized
}
func (c *client) login() error {
// prepare HTTP request
v := url.Values{}
v.Set("username", c.url.User.Username())
passwd, _ := c.url.User.Password()
v.Set("password", passwd)
resp, err := c.post("/api/v2/auth/login", v)
if err != nil {
return err
}
// check result
body := make([]byte, 2)
_, err = resp.Body.Read(body)
if err != nil {
return err
}
if string(body) != "Ok" {
return errors.New("failed to login into qBittorrent webui with url: " + c.url.String())
}
return nil
}
func (c *client) post(path string, data url.Values) (*http.Response, error) {
u := c.url.JoinPath(path)
u.User = nil // remove userinfo for requests
req, err := http.NewRequest("POST", u.String(), bytes.NewReader([]byte(data.Encode())))
if err != nil {
return nil, err
}
if data != nil {
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
}
resp, err := c.client.Do(req)
if err != nil {
return nil, err
}
if resp.Cookies() != nil {
c.client.Jar.SetCookies(u, resp.Cookies())
}
return resp, nil
}
func (c *client) AddFromLink(link string, savePath string, id string) error {
err := c.checkAuthorization()
if err != nil {
return err
}
buf := new(bytes.Buffer)
writer := multipart.NewWriter(buf)
addField := func(name string, value string) {
if err != nil {
return
}
err = writer.WriteField(name, value)
}
addField("urls", link)
addField("savepath", savePath)
addField("tags", "alist-"+id)
addField("autoTMM", "false")
if err != nil {
return err
}
err = writer.Close()
if err != nil {
return err
}
u := c.url.JoinPath("/api/v2/torrents/add")
u.User = nil // remove userinfo for requests
req, err := http.NewRequest("POST", u.String(), buf)
if err != nil {
return err
}
req.Header.Add("Content-Type", writer.FormDataContentType())
resp, err := c.client.Do(req)
if err != nil {
return err
}
// check result
body := make([]byte, 2)
_, err = resp.Body.Read(body)
if err != nil {
return err
}
if resp.StatusCode != 200 || string(body) != "Ok" {
return errors.New("failed to add qBittorrent task: " + link)
}
return nil
}
type TorrentStatus string
const (
ERROR TorrentStatus = "error"
MISSINGFILES TorrentStatus = "missingFiles"
UPLOADING TorrentStatus = "uploading"
PAUSEDUP TorrentStatus = "pausedUP"
QUEUEDUP TorrentStatus = "queuedUP"
STALLEDUP TorrentStatus = "stalledUP"
CHECKINGUP TorrentStatus = "checkingUP"
FORCEDUP TorrentStatus = "forcedUP"
ALLOCATING TorrentStatus = "allocating"
DOWNLOADING TorrentStatus = "downloading"
METADL TorrentStatus = "metaDL"
PAUSEDDL TorrentStatus = "pausedDL"
QUEUEDDL TorrentStatus = "queuedDL"
STALLEDDL TorrentStatus = "stalledDL"
CHECKINGDL TorrentStatus = "checkingDL"
FORCEDDL TorrentStatus = "forcedDL"
CHECKINGRESUMEDATA TorrentStatus = "checkingResumeData"
MOVING TorrentStatus = "moving"
UNKNOWN TorrentStatus = "unknown"
)
// https://github.com/DGuang21/PTGo/blob/main/app/client/client_distributer.go
type TorrentInfo struct {
AddedOn int `json:"added_on"` // 将 torrent 添加到客户端的时间（Unix Epoch）
AmountLeft int64 `json:"amount_left"` // 剩余大小（字节）
AutoTmm bool `json:"auto_tmm"` // 此 torrent 是否由 Automatic Torrent Management 管理
Availability float64 `json:"availability"` // 当前百分比
Category string `json:"category"` //
Completed int64 `json:"completed"` // 完成的传输数据量（字节）
CompletionOn int `json:"completion_on"` // Torrent 完成的时间（Unix Epoch）
ContentPath string `json:"content_path"` // torrent 内容的绝对路径（多文件 torrent 的根路径，单文件 torrent 的绝对文件路径）
DlLimit int `json:"dl_limit"` // Torrent 下载速度限制（字节/秒）
Dlspeed int `json:"dlspeed"` // Torrent 下载速度（字节/秒）
Downloaded int64 `json:"downloaded"` // 已经下载大小
DownloadedSession int64 `json:"downloaded_session"` // 此会话下载的数据量
Eta int `json:"eta"` //
FLPiecePrio bool `json:"f_l_piece_prio"` // 如果第一个最后一块被优先考虑，则为true
ForceStart bool `json:"force_start"` // 如果为此 torrent 启用了强制启动，则为true
Hash string `json:"hash"` //
LastActivity int `json:"last_activity"` // 上次活跃的时间（Unix Epoch）
MagnetURI string `json:"magnet_uri"` // 与此 torrent 对应的 Magnet URI
MaxRatio float64 `json:"max_ratio"` // 种子/上传停止种子前的最大共享比率
MaxSeedingTime int `json:"max_seeding_time"` // 停止种子种子前的最长种子时间（秒）
Name string `json:"name"` //
NumComplete int `json:"num_complete"` //
NumIncomplete int `json:"num_incomplete"` //
NumLeechs int `json:"num_leechs"` // 连接到的 leechers 的数量
NumSeeds int `json:"num_seeds"` // 连接到的种子数
Priority int `json:"priority"` // 速度优先。如果队列被禁用或 torrent 处于种子模式，则返回 -1
Progress float64 `json:"progress"` // 进度
Ratio float64 `json:"ratio"` // Torrent 共享比率
RatioLimit int `json:"ratio_limit"` //
SavePath string `json:"save_path"`
SeedingTime int `json:"seeding_time"` // Torrent 完成用时（秒）
SeedingTimeLimit int `json:"seeding_time_limit"` // max_seeding_time
SeenComplete int `json:"seen_complete"` // 上次 torrent 完成的时间
SeqDl bool `json:"seq_dl"` // 如果启用顺序下载，则为true
Size int64 `json:"size"` //
State TorrentStatus `json:"state"` // 参见https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-(qBittorrent-4.1)#get-torrent-list
SuperSeeding bool `json:"super_seeding"` // 如果启用超级播种，则为true
Tags string `json:"tags"` // Torrent 的逗号连接标签列表
TimeActive int `json:"time_active"` // 总活动时间（秒）
TotalSize int64 `json:"total_size"` // 此 torrent 中所有文件的总大小（字节）（包括未选择的文件）
Tracker string `json:"tracker"` // 第一个具有工作状态的tracker。如果没有tracker在工作，则返回空字符串。
TrackersCount int `json:"trackers_count"` //
UpLimit int `json:"up_limit"` // 上传限制
Uploaded int64 `json:"uploaded"` // 累计上传
UploadedSession int64 `json:"uploaded_session"` // 当前session累计上传
Upspeed int `json:"upspeed"` // 上传速度（字节/秒）
}
type InfoNotFoundError struct {
Id string
Err error
}
func (i InfoNotFoundError) Error() string {
return "there should be exactly one task with tag \"alist-" + i.Id + "\""
}
func NewInfoNotFoundError(id string) InfoNotFoundError {
return InfoNotFoundError{Id: id}
}
func (c *client) GetInfo(id string) (TorrentInfo, error) {
var infos []TorrentInfo
err := c.checkAuthorization()
if err != nil {
return TorrentInfo{}, err
}
v := url.Values{}
v.Set("tag", "alist-"+id)
response, err := c.post("/api/v2/torrents/info", v)
if err != nil {
return TorrentInfo{}, err
}
body, err := io.ReadAll(response.Body)
if err != nil {
return TorrentInfo{}, err
}
err = utils.Json.Unmarshal(body, &infos)
if err != nil {
return TorrentInfo{}, err
}
if len(infos) != 1 {
return TorrentInfo{}, NewInfoNotFoundError(id)
}
return infos[0], nil
}
type FileInfo struct {
Index int `json:"index"`
Name string `json:"name"`
Size int64 `json:"size"`
Progress float32 `json:"progress"`
Priority int `json:"priority"`
IsSeed bool `json:"is_seed"`
PieceRange []int `json:"piece_range"`
Availability float32 `json:"availability"`
}
func (c *client) GetFiles(id string) ([]FileInfo, error) {
var infos []FileInfo
err := c.checkAuthorization()
if err != nil {
return []FileInfo{}, err
}
tInfo, err := c.GetInfo(id)
if err != nil {
return []FileInfo{}, err
}
v := url.Values{}
v.Set("hash", tInfo.Hash)
response, err := c.post("/api/v2/torrents/files", v)
if err != nil {
return []FileInfo{}, err
}
body, err := io.ReadAll(response.Body)
if err != nil {
return []FileInfo{}, err
}
err = utils.Json.Unmarshal(body, &infos)
if err != nil {
return []FileInfo{}, err
}
return infos, nil
}
func (c *client) Delete(id string, deleteFiles bool) error {
err := c.checkAuthorization()
if err != nil {
return err
}
info, err := c.GetInfo(id)
if err != nil {
return err
}
v := url.Values{}
v.Set("hashes", info.Hash)
if deleteFiles {
v.Set("deleteFiles", "true")
} else {
v.Set("deleteFiles", "false")
}
response, err := c.post("/api/v2/torrents/delete", v)
if err != nil {
return err
}
if response.StatusCode != 200 {
return errors.New("failed to delete qbittorrent task")
}
v = url.Values{}
v.Set("tags", "alist-"+id)
response, err = c.post("/api/v2/torrents/deleteTags", v)
if err != nil {
return err
}
if response.StatusCode != 200 {
return errors.New("failed to delete qbittorrent tag")
}
return nil
}

6
pkg/task/task.go
View File

@ -26,7 +26,7 @@ type Task[K comparable] struct {
Name string
state string // pending, running, finished, canceling, canceled, errored
status string
progress int
progress float64
Error error
@ -41,11 +41,11 @@ func (t *Task[K]) SetStatus(status string) {
t.status = status
}
func (t *Task[K]) SetProgress(percentage int) {
func (t *Task[K]) SetProgress(percentage float64) {
t.progress = percentage
}
func (t Task[K]) GetProgress() int {
func (t Task[K]) GetProgress() float64 {
return t.progress
}

7
pkg/utils/io.go
View File

@ -5,10 +5,11 @@ import (
"context"
"errors"
"fmt"
"golang.org/x/exp/constraints"
"io"
"time"
"golang.org/x/exp/constraints"
log "github.com/sirupsen/logrus"
)
@ -21,7 +22,7 @@ func (rf readerFunc) Read(p []byte) (n int, err error) { return rf(p) }
// CopyWithCtx slightly modified function signature:
// - context has been added in order to propagate cancellation
// - I do not return the number of bytes written, has it is not useful in my use case
func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, progress func(percentage int)) error {
func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, progress func(percentage float64)) error {
// Copy will call the Reader and Writer interface multiple time, in order
// to copy by chunk (avoiding loading the whole file in memory).
// I insert the ability to cancel before read time as it is the earliest
@ -40,7 +41,7 @@ func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, p
n, err := in.Read(p)
if s > 0 && (err == nil || err == io.EOF) {
finish += int64(n)
progress(int(finish / s))
progress(float64(finish) / float64(s))
}
return n, err
}