From b60da9732f22b22d84f015d6aaabcb2f058871d1 Mon Sep 17 00:00:00 2001 From: Jealous Date: Fri, 10 Jan 2025 21:24:44 +0800 Subject: [PATCH] feat(offline-download): allow using offline download tools in any storage (#7716) * Feat(offline-download): allow using thunder offline download tool in any storage * Feat(offline-download): allow using 115 offline download tool in any storage * Feat(offline-download): allow using pikpak offline download tool in any storage * style(offline-download): unify offline download tool names * feat(offline-download): show available offline download tools only * Fix(offline-download): update unmodified tool names. --------- Co-authored-by: Andy Hsu --- internal/conf/const.go | 9 + internal/offline_download/115/client.go | 23 +- internal/offline_download/pikpak/pikpak.go | 25 +- internal/offline_download/thunder/thunder.go | 25 +- internal/offline_download/tool/add.go | 33 ++- internal/offline_download/tool/all_test.go | 17 -- internal/offline_download/tool/base.go | 29 -- internal/offline_download/tool/download.go | 55 +--- internal/offline_download/tool/tools.go | 6 +- internal/offline_download/tool/transfer.go | 279 ++++++++++++++---- internal/offline_download/tool/util.go | 41 --- .../offline_download/transmission/client.go | 2 +- server/handles/offline_download.go | 147 ++++++++- server/router.go | 3 + 14 files changed, 484 insertions(+), 210 deletions(-) delete mode 100644 internal/offline_download/tool/all_test.go delete mode 100644 internal/offline_download/tool/util.go diff --git a/internal/conf/const.go b/internal/conf/const.go index 99e8c868..0e534350 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -58,6 +58,15 @@ const ( TransmissionUri = "transmission_uri" TransmissionSeedtime = "transmission_seedtime" + // 115 + Pan115TempDir = "115_temp_dir" + + // pikpak + PikPakTempDir = "pikpak_temp_dir" + + // thunder + ThunderTempDir = "thunder_temp_dir" + // single Token = "token" IndexProgress = "index_progress" diff --git a/internal/offline_download/115/client.go b/internal/offline_download/115/client.go index 45f147db..3f9d804d 100644 --- a/internal/offline_download/115/client.go +++ b/internal/offline_download/115/client.go @@ -3,6 +3,8 @@ package _115 import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/drivers/115" "github.com/alist-org/alist/v3/internal/errs" @@ -33,13 +35,23 @@ func (p *Cloud115) Init() (string, error) { } func (p *Cloud115) IsReady() bool { + tempDir := setting.GetStr(conf.Pan115TempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*_115.Pan115); !ok { + return false + } return true } func (p *Cloud115) AddURL(args *tool.AddUrlArgs) (string, error) { // 添加新任务刷新缓存 p.refreshTaskCache = true - // args.TempDir 已经被修改为了 DstDirPath storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err @@ -50,6 +62,11 @@ func (p *Cloud115) AddURL(args *tool.AddUrlArgs) (string, error) { } ctx := context.Background() + + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err @@ -64,7 +81,7 @@ func (p *Cloud115) AddURL(args *tool.AddUrlArgs) (string, error) { } func (p *Cloud115) Remove(task *tool.DownloadTask) error { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } @@ -81,7 +98,7 @@ func (p *Cloud115) Remove(task *tool.DownloadTask) error { } func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } diff --git a/internal/offline_download/pikpak/pikpak.go b/internal/offline_download/pikpak/pikpak.go index f07b3de8..8fdfb340 100644 --- a/internal/offline_download/pikpak/pikpak.go +++ b/internal/offline_download/pikpak/pikpak.go @@ -3,6 +3,8 @@ package pikpak import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" "strconv" "github.com/alist-org/alist/v3/drivers/pikpak" @@ -17,7 +19,7 @@ type PikPak struct { } func (p *PikPak) Name() string { - return "pikpak" + return "PikPak" } func (p *PikPak) Items() []model.SettingItem { @@ -34,13 +36,23 @@ func (p *PikPak) Init() (string, error) { } func (p *PikPak) IsReady() bool { + tempDir := setting.GetStr(conf.PikPakTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*pikpak.PikPak); !ok { + return false + } return true } func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) { // 添加新任务刷新缓存 p.refreshTaskCache = true - // args.TempDir 已经被修改为了 DstDirPath storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err @@ -51,6 +63,11 @@ func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) { } ctx := context.Background() + + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err @@ -65,7 +82,7 @@ func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) { } func (p *PikPak) Remove(task *tool.DownloadTask) error { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } @@ -82,7 +99,7 @@ func (p *PikPak) Remove(task *tool.DownloadTask) error { } func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } diff --git a/internal/offline_download/thunder/thunder.go b/internal/offline_download/thunder/thunder.go index 3ab8b002..81b94861 100644 --- a/internal/offline_download/thunder/thunder.go +++ b/internal/offline_download/thunder/thunder.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" "strconv" "github.com/alist-org/alist/v3/drivers/thunder" @@ -18,7 +20,7 @@ type Thunder struct { } func (t *Thunder) Name() string { - return "thunder" + return "Thunder" } func (t *Thunder) Items() []model.SettingItem { @@ -35,13 +37,23 @@ func (t *Thunder) Init() (string, error) { } func (t *Thunder) IsReady() bool { + tempDir := setting.GetStr(conf.ThunderTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*thunder.Thunder); !ok { + return false + } return true } func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { // 添加新任务刷新缓存 t.refreshTaskCache = true - // args.TempDir 已经被修改为了 DstDirPath storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err @@ -52,6 +64,11 @@ func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { } ctx := context.Background() + + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err @@ -66,7 +83,7 @@ func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { } func (t *Thunder) Remove(task *tool.DownloadTask) error { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } @@ -83,7 +100,7 @@ func (t *Thunder) Remove(task *tool.DownloadTask) error { } func (t *Thunder) Status(task *tool.DownloadTask) (*tool.Status, error) { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 405f96cb..884e166b 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,8 +2,12 @@ package tool import ( "context" + _115 "github.com/alist-org/alist/v3/drivers/115" + "github.com/alist-org/alist/v3/drivers/pikpak" + "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/internal/task" "net/url" "path" @@ -76,19 +80,26 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) deletePolicy := args.DeletePolicy + // 如果当前 storage 是对应网盘，则直接下载到目标路径，无需转存 switch args.Tool { case "115 Cloud": - tempDir = args.DstDirPath - // 防止将下载好的文件删除 - deletePolicy = DeleteNever - case "pikpak": - tempDir = args.DstDirPath - // 防止将下载好的文件删除 - deletePolicy = DeleteNever - case "thunder": - tempDir = args.DstDirPath - // 防止将下载好的文件删除 - deletePolicy = DeleteNever + if _, ok := storage.(*_115.Pan115); ok { + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.Pan115TempDir), uid) + } + case "PikPak": + if _, ok := storage.(*pikpak.PikPak); ok { + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.PikPakTempDir), uid) + } + case "Thunder": + if _, ok := storage.(*thunder.Thunder); ok { + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.ThunderTempDir), uid) + } } taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed diff --git a/internal/offline_download/tool/all_test.go b/internal/offline_download/tool/all_test.go deleted file mode 100644 index 27da5e32..00000000 --- a/internal/offline_download/tool/all_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package tool_test - -import ( - "testing" - - "github.com/alist-org/alist/v3/internal/offline_download/tool" -) - -func TestGetFiles(t *testing.T) { - files, err := tool.GetFiles("..") - if err != nil { - t.Fatal(err) - } - for _, file := range files { - t.Log(file.Name, file.Size, file.Path, file.Modified) - } -} diff --git a/internal/offline_download/tool/base.go b/internal/offline_download/tool/base.go index ae9eac26..b14169f8 100644 --- a/internal/offline_download/tool/base.go +++ b/internal/offline_download/tool/base.go @@ -1,10 +1,6 @@ package tool import ( - "io" - "os" - "time" - "github.com/alist-org/alist/v3/internal/model" ) @@ -40,28 +36,3 @@ type Tool interface { // Run for simple http download Run(task *DownloadTask) error } - -type GetFileser interface { - // GetFiles return the files of the download task, if nil, means walk the temp dir to get the files - GetFiles(task *DownloadTask) []File -} - -type File struct { - // ReadCloser for http client - ReadCloser io.ReadCloser - Name string - Size int64 - Path string - Modified time.Time -} - -func (f *File) GetReadCloser() (io.ReadCloser, error) { - if f.ReadCloser != nil { - return f.ReadCloser, nil - } - file, err := os.Open(f.Path) - if err != nil { - return nil, err - } - return file, nil -} diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 94bf7dbb..c3b30f1b 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -40,7 +40,7 @@ func (t *DownloadTask) Run() error { } if err := t.tool.Run(t); !errs.IsNotSupportError(err) { if err == nil { - return t.Complete() + return t.Transfer() } return err } @@ -80,10 +80,10 @@ outer: if err != nil { return err } - if t.tool.Name() == "pikpak" { + if t.tool.Name() == "Pikpak" { return nil } - if t.tool.Name() == "thunder" { + if t.tool.Name() == "Thunder" { return nil } if t.tool.Name() == "115 Cloud" { @@ -109,7 +109,7 @@ outer: } } - if t.tool.Name() == "transmission" { + if t.tool.Name() == "Transmission" { // hack for transmission seedTime := setting.GetInt(conf.TransmissionSeedtime, 0) if seedTime >= 0 { @@ -146,7 +146,7 @@ func (t *DownloadTask) Update() (bool, error) { } // if download completed if info.Completed { - err := t.Complete() + err := t.Transfer() return true, errors.WithMessage(err, "failed to transfer file") } // if download failed @@ -156,45 +156,16 @@ func (t *DownloadTask) Update() (bool, error) { return false, nil } -func (t *DownloadTask) Complete() error { - var ( - files []File - err error - ) - if t.tool.Name() == "pikpak" { - return nil - } - if t.tool.Name() == "thunder" { - return nil - } - if t.tool.Name() == "115 Cloud" { - return nil - } - if getFileser, ok := t.tool.(GetFileser); ok { - files = getFileser.GetFiles(t) - } else { - files, err = GetFiles(t.TempDir) - if err != nil { - return errors.Wrapf(err, "failed to get files") +func (t *DownloadTask) Transfer() error { + toolName := t.tool.Name() + if toolName == "115 Cloud" || toolName == "PikPak" || toolName == "Thunder" { + // 如果不是直接下载到目标路径，则进行转存 + if t.TempDir != t.DstDirPath { + return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) } + return nil } - // upload files - for i := range files { - file := files[i] - tsk := &TransferTask{ - TaskExtension: task.TaskExtension{ - Creator: t.GetCreator(), - }, - file: file, - DstDirPath: t.DstDirPath, - TempDir: t.TempDir, - DeletePolicy: t.DeletePolicy, - FileDir: file.Path, - } - tsk.SetTotalBytes(file.Size) - TransferTaskManager.Add(tsk) - } - return nil + return transferStd(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) } func (t *DownloadTask) GetName() string { diff --git a/internal/offline_download/tool/tools.go b/internal/offline_download/tool/tools.go index 9de7d526..4a31ac7f 100644 --- a/internal/offline_download/tool/tools.go +++ b/internal/offline_download/tool/tools.go @@ -3,6 +3,7 @@ package tool import ( "fmt" "github.com/alist-org/alist/v3/internal/model" + "sort" ) var ( @@ -25,8 +26,11 @@ func (t ToolsManager) Add(tool Tool) { func (t ToolsManager) Names() []string { names := make([]string, 0, len(t)) for name := range t { - names = append(names, name) + if tool, err := t.Get(name); err == nil && tool.IsReady() { + names = append(names, name) + } } + sort.Strings(names) return names } diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index a77c4822..8c7ab244 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -1,11 +1,9 @@ package tool import ( + "context" "fmt" - "os" - "path/filepath" - "time" - + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/stream" @@ -14,80 +12,60 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/xhofe/tache" + "net/http" + "os" + stdpath "path" + "path/filepath" + "time" ) type TransferTask struct { task.TaskExtension - FileDir string `json:"file_dir"` - DstDirPath string `json:"dst_dir_path"` - TempDir string `json:"temp_dir"` - DeletePolicy DeletePolicy `json:"delete_policy"` - file File + Status string `json:"-"` //don't save status to save space + SrcObjPath string `json:"src_obj_path"` + DstDirPath string `json:"dst_dir_path"` + SrcStorage driver.Driver `json:"-"` + DstStorage driver.Driver `json:"-"` + SrcStorageMp string `json:"src_storage_mp"` + DstStorageMp string `json:"dst_storage_mp"` + DeletePolicy DeletePolicy `json:"delete_policy"` } func (t *TransferTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() - // check dstDir again - var err error - if (t.file == File{}) { - t.file, err = GetFile(t.FileDir) - if err != nil { - return errors.Wrapf(err, "failed to get file %s", t.FileDir) - } + if t.SrcStorage == nil { + return transferStdPath(t) + } else { + return transferObjPath(t) } - storage, dstDirActualPath, err := op.GetStorageAndActualPath(t.DstDirPath) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - mimetype := utils.GetMimeType(t.file.Path) - rc, err := t.file.GetReadCloser() - if err != nil { - return errors.Wrapf(err, "failed to open file %s", t.file.Path) - } - s := &stream.FileStream{ - Ctx: nil, - Obj: &model.Object{ - Name: filepath.Base(t.file.Path), - Size: t.file.Size, - Modified: t.file.Modified, - IsFolder: false, - }, - Reader: rc, - Mimetype: mimetype, - Closers: utils.NewClosers(rc), - } - relDir, err := filepath.Rel(t.TempDir, filepath.Dir(t.file.Path)) - if err != nil { - log.Errorf("find relation directory error: %v", err) - } - newDistDir := filepath.Join(dstDirActualPath, relDir) - return op.Put(t.Ctx(), storage, newDistDir, s, t.SetProgress) } func (t *TransferTask) GetName() string { - return fmt.Sprintf("transfer %s to [%s]", t.file.Path, t.DstDirPath) + return fmt.Sprintf("transfer [%s](%s) to [%s](%s)", t.SrcStorageMp, t.SrcObjPath, t.DstStorageMp, t.DstDirPath) } func (t *TransferTask) GetStatus() string { - return "transferring" + return t.Status } func (t *TransferTask) OnSucceeded() { if t.DeletePolicy == DeleteOnUploadSucceed || t.DeletePolicy == DeleteAlways { - err := os.Remove(t.file.Path) - if err != nil { - log.Errorf("failed to delete file %s, error: %s", t.file.Path, err.Error()) + if t.SrcStorage == nil { + removeStdTemp(t) + } else { + removeObjTemp(t) } } } func (t *TransferTask) OnFailed() { if t.DeletePolicy == DeleteOnUploadFailed || t.DeletePolicy == DeleteAlways { - err := os.Remove(t.file.Path) - if err != nil { - log.Errorf("failed to delete file %s, error: %s", t.file.Path, err.Error()) + if t.SrcStorage == nil { + removeStdTemp(t) + } else { + removeObjTemp(t) } } } @@ -95,3 +73,202 @@ func (t *TransferTask) OnFailed() { var ( TransferTaskManager *tache.Manager[*TransferTask] ) + +func transferStd(ctx context.Context, tempDir, dstDirPath string, deletePolicy DeletePolicy) error { + dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed get dst storage") + } + entries, err := os.ReadDir(tempDir) + if err != nil { + return err + } + taskCreator, _ := ctx.Value("user").(*model.User) + for _, entry := range entries { + t := &TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: taskCreator, + }, + SrcObjPath: stdpath.Join(tempDir, entry.Name()), + DstDirPath: dstDirActualPath, + DstStorage: dstStorage, + DstStorageMp: dstStorage.GetStorage().MountPath, + DeletePolicy: deletePolicy, + } + TransferTaskManager.Add(t) + } + return nil +} + +func transferStdPath(t *TransferTask) error { + t.Status = "getting src object" + info, err := os.Stat(t.SrcObjPath) + if err != nil { + return err + } + if info.IsDir() { + t.Status = "src object is dir, listing objs" + entries, err := os.ReadDir(t.SrcObjPath) + if err != nil { + return err + } + for _, entry := range entries { + srcRawPath := stdpath.Join(t.SrcObjPath, entry.Name()) + dstObjPath := stdpath.Join(t.DstDirPath, info.Name()) + t := &TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: t.Creator, + }, + SrcObjPath: srcRawPath, + DstDirPath: dstObjPath, + DstStorage: t.DstStorage, + SrcStorageMp: t.SrcStorageMp, + DstStorageMp: t.DstStorageMp, + DeletePolicy: t.DeletePolicy, + } + TransferTaskManager.Add(t) + } + t.Status = "src object is dir, added all transfer tasks of files" + return nil + } + return transferStdFile(t) +} + +func transferStdFile(t *TransferTask) error { + rc, err := os.Open(t.SrcObjPath) + if err != nil { + return errors.Wrapf(err, "failed to open file %s", t.SrcObjPath) + } + info, err := rc.Stat() + if err != nil { + return errors.Wrapf(err, "failed to get file %s", t.SrcObjPath) + } + mimetype := utils.GetMimeType(t.SrcObjPath) + s := &stream.FileStream{ + Ctx: nil, + Obj: &model.Object{ + Name: filepath.Base(t.SrcObjPath), + Size: info.Size(), + Modified: info.ModTime(), + IsFolder: false, + }, + Reader: rc, + Mimetype: mimetype, + Closers: utils.NewClosers(rc), + } + t.SetTotalBytes(info.Size()) + return op.Put(t.Ctx(), t.DstStorage, t.DstDirPath, s, t.SetProgress) +} + +func removeStdTemp(t *TransferTask) { + info, err := os.Stat(t.SrcObjPath) + if err != nil || info.IsDir() { + return + } + if err := os.Remove(t.SrcObjPath); err != nil { + log.Errorf("failed to delete temp file %s, error: %s", t.SrcObjPath, err.Error()) + } +} + +func transferObj(ctx context.Context, tempDir, dstDirPath string, deletePolicy DeletePolicy) error { + srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return errors.WithMessage(err, "failed get src storage") + } + dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed get dst storage") + } + objs, err := op.List(ctx, srcStorage, srcObjActualPath, model.ListArgs{}) + if err != nil { + return errors.WithMessagef(err, "failed list src [%s] objs", tempDir) + } + taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed + for _, obj := range objs { + t := &TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: taskCreator, + }, + SrcObjPath: stdpath.Join(srcObjActualPath, obj.GetName()), + DstDirPath: dstDirActualPath, + SrcStorage: srcStorage, + DstStorage: dstStorage, + SrcStorageMp: srcStorage.GetStorage().MountPath, + DstStorageMp: dstStorage.GetStorage().MountPath, + DeletePolicy: deletePolicy, + } + TransferTaskManager.Add(t) + } + return nil +} + +func transferObjPath(t *TransferTask) error { + t.Status = "getting src object" + srcObj, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcObjPath) + if err != nil { + return errors.WithMessagef(err, "failed get src [%s] file", t.SrcObjPath) + } + if srcObj.IsDir() { + t.Status = "src object is dir, listing objs" + objs, err := op.List(t.Ctx(), t.SrcStorage, t.SrcObjPath, model.ListArgs{}) + if err != nil { + return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcObjPath) + } + for _, obj := range objs { + if utils.IsCanceled(t.Ctx()) { + return nil + } + srcObjPath := stdpath.Join(t.SrcObjPath, obj.GetName()) + dstObjPath := stdpath.Join(t.DstDirPath, srcObj.GetName()) + TransferTaskManager.Add(&TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: t.Creator, + }, + SrcObjPath: srcObjPath, + DstDirPath: dstObjPath, + SrcStorage: t.SrcStorage, + DstStorage: t.DstStorage, + SrcStorageMp: t.SrcStorageMp, + DstStorageMp: t.DstStorageMp, + DeletePolicy: t.DeletePolicy, + }) + } + t.Status = "src object is dir, added all transfer tasks of objs" + return nil + } + return transferObjFile(t) +} + +func transferObjFile(t *TransferTask) error { + srcFile, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcObjPath) + if err != nil { + return errors.WithMessagef(err, "failed get src [%s] file", t.SrcObjPath) + } + link, _, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcObjPath, model.LinkArgs{ + Header: http.Header{}, + }) + if err != nil { + return errors.WithMessagef(err, "failed get [%s] link", t.SrcObjPath) + } + fs := stream.FileStream{ + Obj: srcFile, + Ctx: t.Ctx(), + } + // any link provided is seekable + ss, err := stream.NewSeekableStream(fs, link) + if err != nil { + return errors.WithMessagef(err, "failed get [%s] stream", t.SrcObjPath) + } + t.SetTotalBytes(srcFile.GetSize()) + return op.Put(t.Ctx(), t.DstStorage, t.DstDirPath, ss, t.SetProgress) +} + +func removeObjTemp(t *TransferTask) { + srcObj, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcObjPath) + if err != nil || srcObj.IsDir() { + return + } + if err := op.Remove(t.Ctx(), t.SrcStorage, t.SrcObjPath); err != nil { + log.Errorf("failed to delete temp obj %s, error: %s", t.SrcObjPath, err.Error()) + } +} diff --git a/internal/offline_download/tool/util.go b/internal/offline_download/tool/util.go deleted file mode 100644 index b2c6ec02..00000000 --- a/internal/offline_download/tool/util.go +++ /dev/null @@ -1,41 +0,0 @@ -package tool - -import ( - "os" - "path/filepath" -) - -func GetFiles(dir string) ([]File, error) { - var files []File - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - files = append(files, File{ - Name: info.Name(), - Size: info.Size(), - Path: path, - Modified: info.ModTime(), - }) - } - return nil - }) - if err != nil { - return nil, err - } - return files, nil -} - -func GetFile(path string) (File, error) { - info, err := os.Stat(path) - if err != nil { - return File{}, err - } - return File{ - Name: info.Name(), - Size: info.Size(), - Path: path, - Modified: info.ModTime(), - }, nil -} diff --git a/internal/offline_download/transmission/client.go b/internal/offline_download/transmission/client.go index 4131f3e1..8049afd6 100644 --- a/internal/offline_download/transmission/client.go +++ b/internal/offline_download/transmission/client.go @@ -29,7 +29,7 @@ func (t *Transmission) Run(task *tool.DownloadTask) error { } func (t *Transmission) Name() string { - return "transmission" + return "Transmission" } func (t *Transmission) Items() []model.SettingItem { diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index c7b7af76..24ff7a05 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -1,6 +1,9 @@ package handles import ( + _115 "github.com/alist-org/alist/v3/drivers/115" + "github.com/alist-org/alist/v3/drivers/pikpak" + "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" @@ -73,11 +76,6 @@ func SetQbittorrent(c *gin.Context) { common.SuccessResp(c, "ok") } -func OfflineDownloadTools(c *gin.Context) { - tools := tool.Tools.Names() - common.SuccessResp(c, tools) -} - type SetTransmissionReq struct { Uri string `json:"uri" form:"uri"` Seedtime string `json:"seedtime" form:"seedtime"` @@ -97,7 +95,7 @@ func SetTransmission(c *gin.Context) { common.ErrorResp(c, err, 500) return } - _tool, err := tool.Tools.Get("transmission") + _tool, err := tool.Tools.Get("Transmission") if err != nil { common.ErrorResp(c, err, 500) return @@ -109,6 +107,143 @@ func SetTransmission(c *gin.Context) { common.SuccessResp(c, "ok") } +type Set115Req struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func Set115(c *gin.Context) { + var req Set115Req + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*_115.Pan115); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only 115 Cloud is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.Pan115TempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("115 Cloud") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +type SetPikPakReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func SetPikPak(c *gin.Context) { + var req SetPikPakReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*pikpak.PikPak); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only PikPak is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.PikPakTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("PikPak") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +type SetThunderReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func SetThunder(c *gin.Context) { + var req SetThunderReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*thunder.Thunder); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only Thunder is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.ThunderTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("Thunder") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +func OfflineDownloadTools(c *gin.Context) { + tools := tool.Tools.Names() + common.SuccessResp(c, tools) +} + type AddOfflineDownloadReq struct { Urls []string `json:"urls"` Path string `json:"path"` diff --git a/server/router.go b/server/router.go index 9ff50365..184de51e 100644 --- a/server/router.go +++ b/server/router.go @@ -132,6 +132,9 @@ func admin(g *gin.RouterGroup) { setting.POST("/set_aria2", handles.SetAria2) setting.POST("/set_qbit", handles.SetQbittorrent) setting.POST("/set_transmission", handles.SetTransmission) + setting.POST("/set_115", handles.Set115) + setting.POST("/set_pikpak", handles.SetPikPak) + setting.POST("/set_thunder", handles.SetThunder) // retain /admin/task API to ensure compatibility with legacy automation scripts _task(g.Group("/task"))