ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Squashed commit of the following:

Browse Source 
commit 4fc0a77565
Author: Andy Hsu <i@nn.ci>
Date:   Fri Oct 20 21:06:25 2023 +0800

    fix(baidu_netdisk): upload file > 4GB (close #5392)

commit aaffaee2b5
Author: gmugu <94156510@qq.com>
Date:   Thu Oct 19 19:17:53 2023 +0800

    perf(webdav): support request with cookies (#5391)

commit 8ef8023c20
Author: NewbieOrange <NewbieOrange@users.noreply.github.com>
Date:   Thu Oct 19 19:17:09 2023 +0800

    fix(aliyundrive_open): upload progress for normal upload (#5398)

commit cdfbe6dcf2
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Wed Oct 18 16:27:07 2023 +0800

    fix: hash gcid empty file (#5394)

commit 94d028743a
Author: Andy Hsu <i@nn.ci>
Date:   Sat Oct 14 13:17:51 2023 +0800

    ci: remove `pr-welcome` label when close issue [skip ci]

commit 7f7335435c
Author: itsHenry <2671230065@qq.com>
Date:   Sat Oct 14 13:12:46 2023 +0800

    feat(cloudreve): support thumbnail (#5373 close #5348)

    * feat(cloudreve): support thumbnail

    * chore: remove unnecessary code

commit b9e192b29c
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Thu Oct 12 20:57:12 2023 +0800

    fix(115): limit request rate (#5367 close #5275)

    * fix(115):limit request rate

    * chore(115): fix unit of `limit_rate`

    ---------

    Co-authored-by: Andy Hsu <i@nn.ci>

commit 69a98eaef6
Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Date:   Wed Oct 11 22:01:55 2023 +0800

    fix(deps): update module github.com/aliyun/aliyun-oss-go-sdk to v2.2.9+incompatible (#5141)

    Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

commit 1ebc96a4e5
Author: Andy Hsu <i@nn.ci>
Date:   Tue Oct 10 18:32:00 2023 +0800

    fix(wopan): fatal error concurrent map writes (close #5352)

commit 66e2324cac
Author: Andy Hsu <i@nn.ci>
Date:   Tue Oct 10 18:23:11 2023 +0800

    chore(deps): upgrade dependencies

commit 7600dc28df
Author: Andy Hsu <i@nn.ci>
Date:   Tue Oct 10 18:13:58 2023 +0800

    fix(aliyundrive_open): change default api to raw server (close #5358)

commit 8ef89ad0a4
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Tue Oct 10 18:08:27 2023 +0800

    fix(baidu_netdisk): hash and `error 2` (#5356)

    * fix(baidu):hash and error:2

    * fix:invalid memory address

commit 35d672217d
Author: jeffmingup <1960588251@qq.com>
Date:   Sun Oct 8 19:29:45 2023 +0800

    fix(onedrive_app): incorrect api on `_accessToken` (#5346)

commit 1a283bb272
Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com>
Date:   Fri Oct 6 16:04:39 2023 +0800

    feat(google_drive): add `hash_info`, `ctime`, `thumbnail` (#5334)

commit a008f54f4d
Author: nkh0472 <67589323+nkh0472@users.noreply.github.com>
Date:   Thu Oct 5 13:10:51 2023 +0800

    docs: minor language improvements (#5329) [skip ci]
This commit is contained in:
Andy Hsu
2023-10-20 21:14:15 +08:00
parent 9fb9efb704
commit ce6e486666
23 changed files with 279 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
drivers/baidu_netdisk/driver.go
View File

@ -5,7 +5,6 @@ import (
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"math"
"net/url"
@ -28,10 +27,9 @@ type BaiduNetdisk struct {
Addition
uploadThread int
vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M)
}
const DefaultSliceSize int64 = 4 * utils.MB
func (d *BaiduNetdisk) Config() driver.Config {
return config
}
@ -54,7 +52,11 @@ func (d *BaiduNetdisk) Init(ctx context.Context) error {
"method": "uinfo",
}, nil)
log.Debugf("[baidu] get uinfo: %s", string(res))
return err
if err != nil {
return err
}
d.vipType = utils.Json.Get(res, "vip_type").ToInt()
return nil
}
func (d *BaiduNetdisk) Drop(ctx context.Context) error {
@ -153,20 +155,13 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo
}
streamSize := stream.GetSize()
rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName())
path := encodeURIComponent(rawPath)
path := stdpath.Join(dstDir.GetPath(), stream.GetName())
mtime := stream.ModTime().Unix()
ctime := stream.CreateTime().Unix()
blockList, _ := utils.Json.MarshalToString([]string{contentMd5})
data := fmt.Sprintf("path=%s&size=%d&isdir=0&rtype=3&block_list=%s&local_mtime=%d&local_ctime=%d",
path, streamSize, blockList, mtime, ctime)
params := map[string]string{
"method": "create",
}
log.Debugf("[baidu_netdisk] precreate data: %s", data)
var newFile File
_, err := d.post("/xpan/file", params, data, &newFile)
_, err := d.create(path, streamSize, 0, "", blockList, &newFile, mtime, ctime)
if err != nil {
return nil, err
}
@ -185,17 +180,18 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
}
streamSize := stream.GetSize()
count := int(math.Max(math.Ceil(float64(streamSize)/float64(DefaultSliceSize)), 1))
lastBlockSize := streamSize % DefaultSliceSize
sliceSize := d.getSliceSize()
count := int(math.Max(math.Ceil(float64(streamSize)/float64(sliceSize)), 1))
lastBlockSize := streamSize % sliceSize
if streamSize > 0 && lastBlockSize == 0 {
lastBlockSize = DefaultSliceSize
lastBlockSize = sliceSize
}
//cal md5 for first 256k data
const SliceSize int64 = 256 * 1024
// cal md5
blockList := make([]string, 0, count)
byteSize := DefaultSliceSize
byteSize := sliceSize
fileMd5H := md5.New()
sliceMd5H := md5.New()
sliceMd5H2 := md5.New()
@ -218,9 +214,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil))
sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil))
blockListStr, _ := utils.Json.MarshalToString(blockList)
rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName())
path := encodeURIComponent(rawPath)
path := stdpath.Join(dstDir.GetPath(), stream.GetName())
mtime := stream.ModTime().Unix()
ctime := stream.CreateTime().Unix()
@ -228,13 +222,23 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
// 尝试获取之前的进度
precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5)
if !ok {
data := fmt.Sprintf("path=%s&size=%d&isdir=0&autoinit=1&rtype=3&block_list=%s&content-md5=%s&slice-md5=%s&local_mtime=%d&local_ctime=%d",
path, streamSize, blockListStr, contentMd5, sliceMd5, mtime, ctime)
params := map[string]string{
"method": "precreate",
}
log.Debugf("[baidu_netdisk] precreate data: %s", data)
_, err = d.post("/xpan/file", params, data, &precreateResp)
form := map[string]string{
"path": path,
"size": strconv.FormatInt(streamSize, 10),
"isdir": "0",
"autoinit": "1",
"rtype": "3",
"block_list": blockListStr,
"content-md5": contentMd5,
"slice-md5": sliceMd5,
}
joinTime(form, ctime, mtime)
log.Debugf("[baidu_netdisk] precreate data: %s", form)
_, err = d.postForm("/xpan/file", params, form, &precreateResp)
if err != nil {
return nil, err
}
@ -257,7 +261,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
break
}
i, partseq, offset, byteSize := i, partseq, int64(partseq)*DefaultSliceSize, DefaultSliceSize
i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize
if partseq+1 == count {
byteSize = lastBlockSize
}
@ -290,7 +294,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
// step.3 创建文件
var newFile File
_, err = d.create(rawPath, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime)
_, err = d.create(path, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime)
if err != nil {
return nil, err
}

5
drivers/baidu_netdisk/types.go
View File

@ -1,7 +1,6 @@
package baidu_netdisk
import (
"github.com/alist-org/alist/v3/pkg/utils"
"path"
"strconv"
"time"
@ -71,7 +70,9 @@ func fileToObj(f File) *model.ObjThumb {
Modified: time.Unix(f.LocalMtime, 0),
Ctime: time.Unix(f.LocalCtime, 0),
IsFolder: f.Isdir == 1,
HashInfo: utils.NewHashInfo(utils.MD5, f.Md5),
// 直接获取的MD5是错误的
// HashInfo: utils.NewHashInfo(utils.MD5, f.Md5),
},
Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3},
}

82
drivers/baidu_netdisk/util.go
View File

@ -4,9 +4,7 @@ import (
"errors"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/alist-org/alist/v3/drivers/base"
@ -96,10 +94,10 @@ func (d *BaiduNetdisk) get(pathname string, params map[string]string, resp inter
}, resp)
}
func (d *BaiduNetdisk) post(pathname string, params map[string]string, data interface{}, resp interface{}) ([]byte, error) {
func (d *BaiduNetdisk) postForm(pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) {
return d.request("https://pan.baidu.com/rest/2.0"+pathname, http.MethodPost, func(req *resty.Request) {
req.SetQueryParams(params)
req.SetBody(data)
req.SetFormData(form)
}, resp)
}
@ -154,6 +152,9 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model
//if res.StatusCode() == 302 {
u = res.Header().Get("location")
//}
updateObjMd5(file, "pan.baidu.com", u)
return &model.Link{
URL: u,
Header: http.Header{
@ -176,6 +177,9 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Li
if err != nil {
return nil, err
}
updateObjMd5(file, d.CustomCrackUA, resp.Info[0].Dlink)
return &model.Link{
URL: resp.Info[0].Dlink,
Header: http.Header{
@ -190,29 +194,73 @@ func (d *BaiduNetdisk) manage(opera string, filelist any) ([]byte, error) {
"opera": opera,
}
marshal, _ := utils.Json.MarshalToString(filelist)
data := fmt.Sprintf("async=0&filelist=%s&ondup=fail", marshal)
return d.post("/xpan/file", params, data, nil)
return d.postForm("/xpan/file", params, map[string]string{
"async": "0",
"filelist": marshal,
"ondup": "fail",
}, nil)
}
func (d *BaiduNetdisk) create(path string, size int64, isdir int, uploadid, block_list string, resp any, mtime, ctime int64) ([]byte, error) {
params := map[string]string{
"method": "create",
}
data := ""
if mtime == 0 || ctime == 0 {
data = fmt.Sprintf("path=%s&size=%d&isdir=%d&rtype=3", encodeURIComponent(path), size, isdir)
} else {
data = fmt.Sprintf("path=%s&size=%d&isdir=%d&rtype=3&local_mtime=%d&local_ctime=%d", encodeURIComponent(path), size, isdir, mtime, ctime)
form := map[string]string{
"path": path,
"size": strconv.FormatInt(size, 10),
"isdir": strconv.Itoa(isdir),
"rtype": "3",
}
if mtime != 0 && ctime != 0 {
joinTime(form, ctime, mtime)
}
if uploadid != "" {
data += fmt.Sprintf("&uploadid=%s&block_list=%s", uploadid, block_list)
form["uploadid"] = uploadid
}
return d.post("/xpan/file", params, data, resp)
if block_list != "" {
form["block_list"] = block_list
}
return d.postForm("/xpan/file", params, form, resp)
}
func encodeURIComponent(str string) string {
r := url.QueryEscape(str)
r = strings.ReplaceAll(r, "+", "%20")
return r
func joinTime(form map[string]string, ctime, mtime int64) {
form["local_mtime"] = strconv.FormatInt(mtime, 10)
form["local_ctime"] = strconv.FormatInt(ctime, 10)
}
func updateObjMd5(obj model.Obj, userAgent, u string) {
object := model.GetRawObject(obj)
if object != nil {
req, _ := http.NewRequest(http.MethodHead, u, nil)
req.Header.Add("User-Agent", userAgent)
resp, _ := base.HttpClient.Do(req)
if resp != nil {
contentMd5 := resp.Header.Get("Content-Md5")
object.HashInfo = utils.NewHashInfo(utils.MD5, contentMd5)
}
}
}
const (
DefaultSliceSize int64 = 4 * utils.MB
VipSliceSize = 16 * utils.MB
SVipSliceSize = 32 * utils.MB
)
func (d *BaiduNetdisk) getSliceSize() int64 {
switch d.vipType {
case 1:
return VipSliceSize
case 2:
return SVipSliceSize
default:
return DefaultSliceSize
}
}
// func encodeURIComponent(str string) string {
// r := url.QueryEscape(str)
// r = strings.ReplaceAll(r, "+", "%20")
// return r
// }