ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: refactor task module

Browse Source
This commit is contained in:
Andy Hsu
2023-11-20 18:01:51 +08:00
parent de9647a5fa
commit 11a30c5044
14 changed files with 405 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
internal/fs/copy.go
View File

@ -3,24 +3,39 @@ package fs
import (
"context"
"fmt"
"net/http"
stdpath "path"
"sync/atomic"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/stream"
"github.com/alist-org/alist/v3/pkg/task"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/xhofe/tache"
"net/http"
stdpath "path"
)
var CopyTaskManager = task.NewTaskManager(3, func(tid *uint64) {
atomic.AddUint64(tid, 1)
})
type CopyTask struct {
tache.Base
Status string `json:"status"`
srcStorage, dstStorage driver.Driver
srcObjPath, dstDirPath string
}
func (t *CopyTask) GetName() string {
return fmt.Sprintf("copy [%s](%s) to [%s](%s)",
t.srcStorage.GetStorage().MountPath, t.srcObjPath, t.dstStorage.GetStorage().MountPath, t.dstDirPath)
}
func (t *CopyTask) GetStatus() string {
return t.Status
}
func (t *CopyTask) Run() error {
return copyBetween2Storages(t, t.srcStorage, t.dstStorage, t.srcObjPath, t.dstDirPath)
}
var CopyTaskManager = tache.NewManager[*CopyTask]()
// Copy if in the same storage, call move method
// if not, add copy task
@ -63,59 +78,52 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool
}
}
// not in the same storage
CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{
Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().MountPath, srcObjActualPath, dstStorage.GetStorage().MountPath, dstDirActualPath),
Func: func(task *task.Task[uint64]) error {
return copyBetween2Storages(task, srcStorage, dstStorage, srcObjActualPath, dstDirActualPath)
},
}))
CopyTaskManager.Add(&CopyTask{
srcStorage: srcStorage,
dstStorage: dstStorage,
srcObjPath: srcObjActualPath,
dstDirPath: dstDirActualPath,
})
return true, nil
}
func copyBetween2Storages(t *task.Task[uint64], srcStorage, dstStorage driver.Driver, srcObjPath, dstDirPath string) error {
t.SetStatus("getting src object")
srcObj, err := op.Get(t.Ctx, srcStorage, srcObjPath)
func copyBetween2Storages(t *CopyTask, srcStorage, dstStorage driver.Driver, srcObjPath, dstDirPath string) error {
t.Status = "getting src object"
srcObj, err := op.Get(t.Ctx(), srcStorage, srcObjPath)
if err != nil {
return errors.WithMessagef(err, "failed get src [%s] file", srcObjPath)
}
if srcObj.IsDir() {
t.SetStatus("src object is dir, listing objs")
objs, err := op.List(t.Ctx, srcStorage, srcObjPath, model.ListArgs{})
t.Status = "src object is dir, listing objs"
objs, err := op.List(t.Ctx(), srcStorage, srcObjPath, model.ListArgs{})
if err != nil {
return errors.WithMessagef(err, "failed list src [%s] objs", srcObjPath)
}
for _, obj := range objs {
if utils.IsCanceled(t.Ctx) {
if utils.IsCanceled(t.Ctx()) {
return nil
}
srcObjPath := stdpath.Join(srcObjPath, obj.GetName())
dstObjPath := stdpath.Join(dstDirPath, srcObj.GetName())
CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{
Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().MountPath, srcObjPath, dstStorage.GetStorage().MountPath, dstObjPath),
Func: func(t *task.Task[uint64]) error {
return copyBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstObjPath)
},
}))
CopyTaskManager.Add(&CopyTask{
srcStorage: srcStorage,
dstStorage: dstStorage,
srcObjPath: srcObjPath,
dstDirPath: dstObjPath,
})
}
} else {
CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{
Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().MountPath, srcObjPath, dstStorage.GetStorage().MountPath, dstDirPath),
Func: func(t *task.Task[uint64]) error {
err := copyFileBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstDirPath)
log.Debugf("copy file between storages: %+v", err)
return err
},
}))
t.Status = "src object is dir, added all copy tasks of objs"
return nil
}
return nil
return copyFileBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstDirPath)
}
func copyFileBetween2Storages(tsk *task.Task[uint64], srcStorage, dstStorage driver.Driver, srcFilePath, dstDirPath string) error {
srcFile, err := op.Get(tsk.Ctx, srcStorage, srcFilePath)
func copyFileBetween2Storages(tsk *CopyTask, srcStorage, dstStorage driver.Driver, srcFilePath, dstDirPath string) error {
srcFile, err := op.Get(tsk.Ctx(), srcStorage, srcFilePath)
if err != nil {
return errors.WithMessagef(err, "failed get src [%s] file", srcFilePath)
}
link, _, err := op.Link(tsk.Ctx, srcStorage, srcFilePath, model.LinkArgs{
link, _, err := op.Link(tsk.Ctx(), srcStorage, srcFilePath, model.LinkArgs{
Header: http.Header{},
})
if err != nil {
@ -123,12 +131,12 @@ func copyFileBetween2Storages(tsk *task.Task[uint64], srcStorage, dstStorage dri
}
fs := stream.FileStream{
Obj: srcFile,
Ctx: tsk.Ctx,
Ctx: tsk.Ctx(),
}
// any link provided is seekable
ss, err := stream.NewSeekableStream(fs, link)
if err != nil {
return errors.WithMessagef(err, "failed get [%s] stream", srcFilePath)
}
return op.Put(tsk.Ctx, dstStorage, dstDirPath, ss, tsk.SetProgress, true)
return op.Put(tsk.Ctx(), dstStorage, dstDirPath, ss, tsk.SetProgress, true)
}