ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(drivers): fs operations and cache (#4965)

Browse Source 
* perf(baidu_photo):multi-thread upload

* perf(baidu_netdisk):multi-thread upload and cache optimization

* fix:LimitWriter

* fix(weiyun):only one login is allowed

* feat(189pc):multi threaded upload

* feat(baidu_netdisk):multi threaded upload

* feat(baidu_photo):multi threaded upload

* feat(weiyun):multi threaded upload

* perf(aliyundriver_open):optimize upload code and optimize cache

* fix(weiyun):invalid directory ID

* fix(baidu_netdisk):modified time

* fix(baidu_netdisk,baidu_photo):upload slice error

* perf(baidu_netdisk):cancel unnecessary retries

* fix(limitWriter):must return a non-nil error if it returns n < len(p)

* fix(aliyundrive_open):Name and Filename only use one

* perf(mopan):multi-thread upload
This commit is contained in:
foxxorcat
2023-08-09 16:13:09 +08:00
committed by GitHub
parent 9d45718e5f
commit df6b306fce
22 changed files with 650 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

216
drivers/baidu_netdisk/driver.go
View File

@ -5,24 +5,28 @@ import (
"crypto/md5"
"encoding/hex"
"fmt"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/avast/retry-go"
log "github.com/sirupsen/logrus"
"io"
"math"
"os"
stdpath "path"
"strconv"
"strings"
"time"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/errgroup"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/avast/retry-go"
log "github.com/sirupsen/logrus"
)
type BaiduNetdisk struct {
model.Storage
Addition
uploadThread int
}
const BaiduFileAPI = "https://d.pcs.baidu.com/rest/2.0/pcs/superfile2"
@ -37,6 +41,10 @@ func (d *BaiduNetdisk) GetAddition() driver.Additional {
}
func (d *BaiduNetdisk) Init(ctx context.Context) error {
d.uploadThread, _ = strconv.Atoi(d.UploadThread)
if d.uploadThread < 1 || d.uploadThread > 32 {
d.uploadThread, d.UploadThread = 3, "3"
}
res, err := d.get("/xpan/nas", map[string]string{
"method": "uinfo",
}, nil)
@ -65,12 +73,16 @@ func (d *BaiduNetdisk) Link(ctx context.Context, file model.Obj, args model.Link
return d.linkOfficial(file, args)
}
func (d *BaiduNetdisk) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
_, err := d.create(stdpath.Join(parentDir.GetPath(), dirName), 0, 1, "", "")
return err
func (d *BaiduNetdisk) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) {
var newDir File
_, err := d.create(stdpath.Join(parentDir.GetPath(), dirName), 0, 1, "", "", &newDir)
if err != nil {
return nil, err
}
return fileToObj(newDir), nil
}
func (d *BaiduNetdisk) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
func (d *BaiduNetdisk) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
data := []base.Json{
{
"path": srcObj.GetPath(),
@ -79,10 +91,18 @@ func (d *BaiduNetdisk) Move(ctx context.Context, srcObj, dstDir model.Obj) error
},
}
_, err := d.manage("move", data)
return err
if err != nil {
return nil, err
}
if srcObj, ok := srcObj.(*model.ObjThumb); ok {
srcObj.SetPath(stdpath.Join(dstDir.GetPath(), srcObj.GetName()))
srcObj.Modified = time.Now()
return srcObj, nil
}
return nil, nil
}
func (d *BaiduNetdisk) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
func (d *BaiduNetdisk) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) {
data := []base.Json{
{
"path": srcObj.GetPath(),
@ -90,7 +110,17 @@ func (d *BaiduNetdisk) Rename(ctx context.Context, srcObj model.Obj, newName str
},
}
_, err := d.manage("rename", data)
return err
if err != nil {
return nil, err
}
if srcObj, ok := srcObj.(*model.ObjThumb); ok {
srcObj.SetPath(stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName))
srcObj.Name = newName
srcObj.Modified = time.Now()
return srcObj, nil
}
return nil, nil
}
func (d *BaiduNetdisk) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
@ -111,63 +141,58 @@ func (d *BaiduNetdisk) Remove(ctx context.Context, obj model.Obj) error {
return err
}
func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
streamSize := stream.GetSize()
func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
tempFile, err := utils.CreateTempFile(stream.GetReadCloser(), stream.GetSize())
if err != nil {
return err
return nil, err
}
defer func() {
_ = tempFile.Close()
_ = os.Remove(tempFile.Name())
}()
count := int(math.Ceil(float64(streamSize) / float64(DefaultSliceSize)))
streamSize := stream.GetSize()
count := int(math.Max(math.Ceil(float64(streamSize)/float64(DefaultSliceSize)), 1))
lastBlockSize := streamSize % DefaultSliceSize
if streamSize > 0 && lastBlockSize == 0 {
lastBlockSize = DefaultSliceSize
}
//cal md5 for first 256k data
const SliceSize int64 = 256 * 1024
// cal md5
h1 := md5.New()
h2 := md5.New()
blockList := make([]string, 0)
contentMd5 := ""
sliceMd5 := ""
left := streamSize
for i := 0; i < count; i++ {
byteSize := DefaultSliceSize
if left < DefaultSliceSize {
byteSize = left
blockList := make([]string, 0, count)
byteSize := DefaultSliceSize
fileMd5H := md5.New()
sliceMd5H := md5.New()
sliceMd5H2 := md5.New()
slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize)
for i := 1; i <= count; i++ {
if utils.IsCanceled(ctx) {
return nil, ctx.Err()
}
left -= byteSize
_, err = io.Copy(io.MultiWriter(h1, h2), io.LimitReader(tempFile, byteSize))
if err != nil {
return err
if i == count {
byteSize = lastBlockSize
}
blockList = append(blockList, fmt.Sprintf("\"%s\"", hex.EncodeToString(h2.Sum(nil))))
h2.Reset()
}
contentMd5 = hex.EncodeToString(h1.Sum(nil))
_, err = tempFile.Seek(0, io.SeekStart)
if err != nil {
return err
}
if streamSize <= SliceSize {
sliceMd5 = contentMd5
} else {
sliceData := make([]byte, SliceSize)
_, err = io.ReadFull(tempFile, sliceData)
if err != nil {
return err
_, err := io.CopyN(io.MultiWriter(fileMd5H, sliceMd5H, slicemd5H2Write), tempFile, byteSize)
if err != nil && err != io.EOF {
return nil, err
}
h2.Write(sliceData)
sliceMd5 = hex.EncodeToString(h2.Sum(nil))
blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil)))
sliceMd5H.Reset()
}
contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil))
sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil))
blockListStr, _ := utils.Json.MarshalToString(blockList)
// step.1 预上传
rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName())
path := encodeURIComponent(rawPath)
block_list_str := fmt.Sprintf("[%s]", strings.Join(blockList, ","))
data := fmt.Sprintf("path=%s&size=%d&isdir=0&autoinit=1&block_list=%s&content-md5=%s&slice-md5=%s",
data := fmt.Sprintf("path=%s&size=%d&isdir=0&autoinit=1&rtype=3&block_list=%s&content-md5=%s&slice-md5=%s",
path, streamSize,
block_list_str,
blockListStr,
contentMd5, sliceMd5)
params := map[string]string{
"method": "precreate",
@ -176,52 +201,65 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
var precreateResp PrecreateResp
_, err = d.post("/xpan/file", params, data, &precreateResp)
if err != nil {
return err
return nil, err
}
log.Debugf("%+v", precreateResp)
if precreateResp.ReturnType == 2 {
//rapid upload, since got md5 match from baidu server
return nil
}
params = map[string]string{
"method": "upload",
"access_token": d.AccessToken,
"type": "tmpfile",
"path": path,
"uploadid": precreateResp.Uploadid,
}
var offset int64 = 0
for i, partseq := range precreateResp.BlockList {
params["partseq"] = strconv.Itoa(partseq)
byteSize := int64(math.Min(float64(streamSize-offset), float64(DefaultSliceSize)))
err := retry.Do(func() error {
return d.uploadSlice(ctx, &params, stream.GetName(), tempFile, offset, byteSize)
},
retry.Context(ctx),
retry.Attempts(3))
if err != nil {
return err
}
offset += byteSize
if len(precreateResp.BlockList) > 0 {
up(i * 100 / len(precreateResp.BlockList))
return nil, err
}
return fileToObj(precreateResp.File), nil
}
_, err = d.create(rawPath, streamSize, 0, precreateResp.Uploadid, block_list_str)
return err
}
func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params *map[string]string, fileName string, file *os.File, offset int64, byteSize int64) error {
_, err := file.Seek(offset, io.SeekStart)
// step.2 上传分片
threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread,
retry.Attempts(3),
retry.Delay(time.Second),
retry.DelayType(retry.BackOffDelay))
for _, partseq := range precreateResp.BlockList {
if utils.IsCanceled(upCtx) {
break
}
partseq, offset, byteSize := partseq, int64(partseq)*DefaultSliceSize, DefaultSliceSize
if partseq+1 == count {
byteSize = lastBlockSize
}
threadG.Go(func(ctx context.Context) error {
params := map[string]string{
"method": "upload",
"access_token": d.AccessToken,
"type": "tmpfile",
"path": path,
"uploadid": precreateResp.Uploadid,
"partseq": strconv.Itoa(partseq),
}
err := d.uploadSlice(ctx, params, stream.GetName(), io.NewSectionReader(tempFile, offset, byteSize))
if err != nil {
return err
}
up(int(threadG.Success()) * 100 / len(precreateResp.BlockList))
return nil
})
}
if err = threadG.Wait(); err != nil {
return nil, err
}
// step.3 创建文件
var newFile File
_, err = d.create(rawPath, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile)
if err != nil {
return err
return nil, err
}
return fileToObj(newFile), nil
}
func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error {
res, err := base.RestyClient.R().
SetContext(ctx).
SetQueryParams(*params).
SetFileReader("file", fileName, io.LimitReader(file, byteSize)).
SetQueryParams(params).
SetFileReader("file", fileName, file).
Post(BaiduFileAPI)
if err != nil {
return err

1
drivers/baidu_netdisk/meta.go
View File

@ -15,6 +15,7 @@ type Addition struct {
ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"`
CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"`
AccessToken string
UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"`
}
var config = driver.Config{

46
drivers/baidu_netdisk/types.go
View File

@ -1,6 +1,7 @@
package baidu_netdisk
import (
"path"
"strconv"
"time"
@ -17,10 +18,8 @@ type File struct {
//OwnerType int `json:"owner_type"`
//Category int `json:"category"`
//RealCategory string `json:"real_category"`
FsId int64 `json:"fs_id"`
ServerMtime int64 `json:"server_mtime"`
FsId int64 `json:"fs_id"`
//OperId int `json:"oper_id"`
//ServerCtime int `json:"server_ctime"`
Thumbs struct {
//Icon string `json:"icon"`
Url3 string `json:"url3"`
@ -28,25 +27,41 @@ type File struct {
//Url1 string `json:"url1"`
} `json:"thumbs"`
//Wpfile int `json:"wpfile"`
//LocalMtime int `json:"local_mtime"`
Size int64 `json:"size"`
//ExtentTinyint7 int `json:"extent_tinyint7"`
Path string `json:"path"`
//Share int `json:"share"`
//ServerAtime int `json:"server_atime"`
//Pl int `json:"pl"`
//LocalCtime int `json:"local_ctime"`
ServerFilename string `json:"server_filename"`
//Md5 string `json:"md5"`
Md5 string `json:"md5"`
//OwnerId int `json:"owner_id"`
//Unlist int `json:"unlist"`
Isdir int `json:"isdir"`
// list resp
//ServerCtime int64 `json:"server_ctime"`
ServerMtime int64 `json:"server_mtime"`
//ServerAtime int64 `json:"server_atime"`
//LocalCtime int64 `json:"local_ctime"`
//LocalMtime int64 `json:"local_mtime"`
// only create and precreate resp
Ctime int64 `json:"ctime"`
Mtime int64 `json:"mtime"`
}
func fileToObj(f File) *model.ObjThumb {
if f.ServerFilename == "" {
f.ServerFilename = path.Base(f.Path)
}
if f.ServerMtime == 0 {
f.ServerMtime = int64(f.Mtime)
}
return &model.ObjThumb{
Object: model.Object{
ID: strconv.FormatInt(f.FsId, 10),
Path: f.Path,
Name: f.ServerFilename,
Size: f.Size,
Modified: time.Unix(f.ServerMtime, 0),
@ -154,10 +169,15 @@ type DownloadResp2 struct {
}
type PrecreateResp struct {
Path string `json:"path"`
Uploadid string `json:"uploadid"`
ReturnType int `json:"return_type"`
BlockList []int `json:"block_list"`
Errno int `json:"errno"`
RequestId int64 `json:"request_id"`
Errno int `json:"errno"`
RequestId int64 `json:"request_id"`
ReturnType int `json:"return_type"`
// return_type=1
Path string `json:"path"`
Uploadid string `json:"uploadid"`
BlockList []int `json:"block_list"`
// return_type=2
File File `json:"info"`
}

28
drivers/baidu_netdisk/util.go
View File

@ -2,17 +2,18 @@ package baidu_netdisk
import (
"fmt"
"github.com/avast/retry-go"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/avast/retry-go"
"github.com/go-resty/resty/v2"
log "github.com/sirupsen/logrus"
)
@ -76,12 +77,20 @@ func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCall
return err2
}
}
return fmt.Errorf("req: [%s] ,errno: %d, refer to https://pan.baidu.com/union/doc/", furl, errno)
err2 := fmt.Errorf("req: [%s] ,errno: %d, refer to https://pan.baidu.com/union/doc/", furl, errno)
if !utils.SliceContains([]int{2}, errno) {
err2 = retry.Unrecoverable(err2)
}
return err2
}
result = res.Body()
return nil
},
retry.Attempts(3))
retry.LastErrorOnly(true),
retry.Attempts(5),
retry.Delay(time.Second),
retry.DelayType(retry.BackOffDelay))
return result, err
}
@ -179,20 +188,17 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Li
}, nil
}
func (d *BaiduNetdisk) manage(opera string, filelist interface{}) ([]byte, error) {
func (d *BaiduNetdisk) manage(opera string, filelist any) ([]byte, error) {
params := map[string]string{
"method": "filemanager",
"opera": opera,
}
marshal, err := utils.Json.Marshal(filelist)
if err != nil {
return nil, err
}
data := fmt.Sprintf("async=0&filelist=%s&ondup=newcopy", string(marshal))
marshal, _ := utils.Json.MarshalToString(filelist)
data := fmt.Sprintf("async=0&filelist=%s&ondup=fail", marshal)
return d.post("/xpan/file", params, data, nil)
}
func (d *BaiduNetdisk) create(path string, size int64, isdir int, uploadid, block_list string) ([]byte, error) {
func (d *BaiduNetdisk) create(path string, size int64, isdir int, uploadid, block_list string, resp any) ([]byte, error) {
params := map[string]string{
"method": "create",
}
@ -200,7 +206,7 @@ func (d *BaiduNetdisk) create(path string, size int64, isdir int, uploadid, bloc
if uploadid != "" {
data += fmt.Sprintf("&uploadid=%s&block_list=%s", uploadid, block_list)
}
return d.post("/xpan/file", params, data, nil)
return d.post("/xpan/file", params, data, resp)
}
func encodeURIComponent(str string) string {