From bf0ee3d31501a2963ebaffbb90b6cb50bdaf7bae Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Mon, 9 May 2022 12:43:51 +0800 Subject: [PATCH 01/16] refactor(baidu.photo): add a file api of download --- drivers/baiduphoto/driver.go | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/drivers/baiduphoto/driver.go b/drivers/baiduphoto/driver.go index 99010ac1..ce871019 100644 --- a/drivers/baiduphoto/driver.go +++ b/drivers/baiduphoto/driver.go @@ -43,6 +43,14 @@ func (driver Baidu) Items() []base.Item { Label: "album_id", Type: base.TypeString, }, + { + Name: "internal_type", + Label: "download api", + Type: base.TypeSelect, + Required: true, + Values: "file,album", + Default: "album", + }, { Name: "client_id", Label: "client id", @@ -156,6 +164,13 @@ func (driver Baidu) Files(path string, account *model.Account) ([]model.File, er } func (driver Baidu) Link(args base.Args, account *model.Account) (*base.Link, error) { + if account.InternalType == "file" { + return driver.LinkFile(args, account) + } + return driver.LinkAlbum(args, account) +} + +func (driver Baidu) LinkAlbum(args base.Args, account *model.Account) (*base.Link, error) { file, err := driver.File(args.Path, account) if err != nil { return nil, err @@ -190,6 +205,43 @@ func (driver Baidu) Link(args base.Args, account *model.Account) (*base.Link, er }, nil } +func (driver Baidu) LinkFile(args base.Args, account *model.Account) (*base.Link, error) { + file, err := driver.File(args.Path, account) + if err != nil { + return nil, err + } + + if !IsAlbumFile(file) { + return nil, base.ErrNotSupport + } + + album, err := driver.File(utils.Dir(utils.ParsePath(args.Path)), account) + if err != nil { + return nil, err + } + // 拷贝到根目录 + cfile, err := driver.CopyAlbumFile(album.Id, account, file.Id) + if err != nil { + return nil, err + } + // 获取文件下载地址 + res, err := base.NoRedirectClient.R(). + SetQueryParams(map[string]string{ + "access_token": account.AccessToken, + "fsid": fmt.Sprint(cfile.Fsid), + }). + Head(FILE_API_URL + "/download") + if err != nil { + return nil, err + } + return &base.Link{ + Headers: []base.Header{ + {Name: "User-Agent", Value: base.UserAgent}, + }, + Url: res.Header().Get("location"), + }, nil +} + func (driver Baidu) Path(path string, account *model.Account) (*model.File, []model.File, error) { path = utils.ParsePath(path) file, err := driver.File(path, account) From 52dcbfe1a4bfc2e772530328226011a616acb044 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Mon, 9 May 2022 18:14:52 +0800 Subject: [PATCH 02/16] fix(xunlei):missing x-client-id error in some user requests --- drivers/xunlei/driver.go | 4 +++- drivers/xunlei/xunlei.go | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/xunlei/driver.go b/drivers/xunlei/driver.go index bd2dddbc..2e5b6325 100644 --- a/drivers/xunlei/driver.go +++ b/drivers/xunlei/driver.go @@ -105,7 +105,7 @@ func (driver XunLeiCloud) Items() []base.Item { Label: "device id", Default: utils.GetMD5Encode(uuid.NewString()), Type: base.TypeString, - Required: false, + Required: true, }, } } @@ -118,8 +118,10 @@ func (driver XunLeiCloud) Save(account *model.Account, old *model.Account) error client := GetClient(account) // 指定验证通过的captchaToken if client.captchaToken != "" { + client.Lock() client.captchaToken = account.CaptchaToken account.CaptchaToken = "" + client.Unlock() } if client.token == "" { diff --git a/drivers/xunlei/xunlei.go b/drivers/xunlei/xunlei.go index 13acd202..a1b1a656 100644 --- a/drivers/xunlei/xunlei.go +++ b/drivers/xunlei/xunlei.go @@ -73,6 +73,8 @@ func (c *Client) requestCaptchaToken(action string, meta map[string]string) erro SetBody(¶m). SetError(&e). SetResult(&resp). + SetHeader("X-Device-Id", c.deviceID). + SetQueryParam("client_id", c.clientID). Post(XLUSER_API_URL + "/shield/captcha/init") if err != nil { return err @@ -133,6 +135,8 @@ func (c *Client) Login(account *model.Account) (err error) { Username: account.Username, Password: account.Password, }). + SetHeader("X-Device-Id", c.deviceID). + SetQueryParam("client_id", c.clientID). Post(url) if err != nil { return err @@ -184,6 +188,8 @@ func (c *Client) RefreshToken() error { "client_id": c.clientID, "client_secret": c.clientSecret, }). + SetHeader("X-Device-Id", c.deviceID). + SetQueryParam("client_id", c.clientID). Post(XLUSER_API_URL + "/auth/token") if err != nil { return err @@ -211,7 +217,8 @@ func (c *Client) Request(method string, url string, callback func(*resty.Request "X-Captcha-Token": c.captchaToken, "User-Agent": c.userAgent, "client_id": c.clientID, - }).SetQueryParam("client_id", c.clientID) + }). + SetQueryParam("client_id", c.clientID) if callback != nil { callback(req) } From d5626d6e2feb15dc16d66687125e2b448d7dcb75 Mon Sep 17 00:00:00 2001 From: Xhofe Date: Tue, 10 May 2022 18:16:32 +0800 Subject: [PATCH 03/16] fix: cancel QueryEscape Disposition (close #1074) --- server/common/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/common/proxy.go b/server/common/proxy.go index 6a84d78f..e6a2174e 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -26,7 +26,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *base.Link, file *model. _ = link.Data.Close() }() w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename=%s`, url.QueryEscape(file.Name))) + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename=%s`, file.Name)) w.Header().Set("Content-Length", strconv.FormatInt(file.Size, 10)) if link.Header != nil { for h, val := range link.Header { From d780fa18a5069e2feed36a696946e880479d78c3 Mon Sep 17 00:00:00 2001 From: Xhofe Date: Tue, 10 May 2022 18:18:06 +0800 Subject: [PATCH 04/16] fix(sftp): error while has no files(close #1078) --- drivers/sftp/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/sftp/driver.go b/drivers/sftp/driver.go index 1b61b536..a0b70b1b 100644 --- a/drivers/sftp/driver.go +++ b/drivers/sftp/driver.go @@ -116,7 +116,7 @@ func (driver SFTP) Files(path string, account *model.Account) ([]model.File, err if err != nil { return nil, err } - var files []model.File + files := make([]model.File, 0) rawFiles, err := client.Files(remotePath) if err != nil { return nil, err From a9027c0f06bd1610c40f2b175fd5c688f42f8191 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Tue, 10 May 2022 11:43:54 +0800 Subject: [PATCH 05/16] fix(baidu.photo):update download api --- drivers/baiduphoto/baidu.go | 2 +- drivers/baiduphoto/driver.go | 19 +++++++++---------- drivers/baiduphoto/types.go | 2 +- drivers/baiduphoto/util.go | 7 ++++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/baiduphoto/baidu.go b/drivers/baiduphoto/baidu.go index 287ec433..c192939b 100644 --- a/drivers/baiduphoto/baidu.go +++ b/drivers/baiduphoto/baidu.go @@ -87,7 +87,7 @@ func (driver Baidu) GetAllFile(account *model.Account) (files []File, err error) for { var resp FileListResp - _, err = driver.Request(http.MethodGet, FILE_API_URL+"/list", func(r *resty.Request) { + _, err = driver.Request(http.MethodGet, FILE_API_URL_V1+"/list", func(r *resty.Request) { r.SetQueryParams(map[string]string{ "need_thumbnail": "1", "need_filter_hidden": "0", diff --git a/drivers/baiduphoto/driver.go b/drivers/baiduphoto/driver.go index ce871019..833ef89f 100644 --- a/drivers/baiduphoto/driver.go +++ b/drivers/baiduphoto/driver.go @@ -224,13 +224,12 @@ func (driver Baidu) LinkFile(args base.Args, account *model.Account) (*base.Link if err != nil { return nil, err } - // 获取文件下载地址 - res, err := base.NoRedirectClient.R(). - SetQueryParams(map[string]string{ - "access_token": account.AccessToken, - "fsid": fmt.Sprint(cfile.Fsid), - }). - Head(FILE_API_URL + "/download") + + res, err := driver.Request(http.MethodGet, FILE_API_URL_V2+"/download", func(r *resty.Request) { + r.SetQueryParams(map[string]string{ + "fsid": fmt.Sprint(cfile.Fsid), + }) + }, account) if err != nil { return nil, err } @@ -238,7 +237,7 @@ func (driver Baidu) LinkFile(args base.Args, account *model.Account) (*base.Link Headers: []base.Header{ {Name: "User-Agent", Value: base.UserAgent}, }, - Url: res.Header().Get("location"), + Url: utils.Json.Get(res.Body(), "dlink").ToString(), }, nil } @@ -454,7 +453,7 @@ func (driver Baidu) Upload(file *model.FileStream, account *model.Account) error // 预上传 var precreateResp PrecreateResp - _, err = driver.Request(http.MethodPost, FILE_API_URL+"/precreate", func(r *resty.Request) { + _, err = driver.Request(http.MethodPost, FILE_API_URL_V1+"/precreate", func(r *resty.Request) { r.SetFormData(params) r.SetResult(&precreateResp) }, account) @@ -483,7 +482,7 @@ func (driver Baidu) Upload(file *model.FileStream, account *model.Account) error fallthrough case 2: // 创建文件 params["uploadid"] = precreateResp.UploadID - _, err = driver.Request(http.MethodPost, FILE_API_URL+"/create", func(r *resty.Request) { + _, err = driver.Request(http.MethodPost, FILE_API_URL_V1+"/create", func(r *resty.Request) { r.SetFormData(params) r.SetResult(&precreateResp) }, account) diff --git a/drivers/baiduphoto/types.go b/drivers/baiduphoto/types.go index 734b5a53..926329d6 100644 --- a/drivers/baiduphoto/types.go +++ b/drivers/baiduphoto/types.go @@ -97,7 +97,7 @@ type ( type ( UploadFile struct { FsID int64 `json:"fs_id"` - Size int `json:"size"` + Size int64 `json:"size"` Md5 string `json:"md5"` ServerFilename string `json:"server_filename"` Path string `json:"path"` diff --git a/drivers/baiduphoto/util.go b/drivers/baiduphoto/util.go index 5ab40a52..2c0542c7 100644 --- a/drivers/baiduphoto/util.go +++ b/drivers/baiduphoto/util.go @@ -13,9 +13,10 @@ import ( ) const ( - API_URL = "https://photo.baidu.com/youai" - ALBUM_API_URL = API_URL + "/album/v1" - FILE_API_URL = API_URL + "/file/v1" + API_URL = "https://photo.baidu.com/youai" + ALBUM_API_URL = API_URL + "/album/v1" + FILE_API_URL_V1 = API_URL + "/file/v1" + FILE_API_URL_V2 = API_URL + "/file/v2" ) var ( From b6af9aa587a6305dd2eeb1847a7d398c6e579464 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Tue, 10 May 2022 21:35:42 +0800 Subject: [PATCH 06/16] fix(139,189,189pc,alidrive,onedrive,yandex):http response body is not close #1072 --- drivers/139/driver.go | 1 + drivers/189/189.go | 1 + drivers/189pc/driver.go | 4 ++++ drivers/alidrive/driver.go | 1 + drivers/onedrive/onedrive.go | 2 ++ drivers/yandex/driver.go | 3 ++- 6 files changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index b0025fcc..8864787c 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -448,6 +448,7 @@ func (driver Cloud139) Upload(file *model.FileStream, account *model.Account) er return err } log.Debugf("%+v", res) + res.Body.Close() start += byteSize } return nil diff --git a/drivers/189/189.go b/drivers/189/189.go index f921a749..ac6ea189 100644 --- a/drivers/189/189.go +++ b/drivers/189/189.go @@ -577,6 +577,7 @@ func (driver Cloud189) NewUpload(file *model.FileStream, account *model.Account) r, err := base.HttpClient.Do(req) log.Debugf("%+v %+v", r, r.Request.Header) + r.Body.Close() if err != nil { return err } diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index b6ca7a1b..22b6edeb 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -619,8 +619,10 @@ func (driver Cloud189) CommonUpload(file *model.FileStream, parentFile *model.Fi } if r.StatusCode != http.StatusOK { data, _ := io.ReadAll(r.Body) + r.Body.Close() return fmt.Errorf(string(data)) } + r.Body.Close() } fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil))) @@ -727,8 +729,10 @@ func (driver Cloud189) FastUpload(file *model.FileStream, parentFile *model.File } if r.StatusCode != http.StatusOK { data, _ := io.ReadAll(r.Body) + r.Body.Close() return fmt.Errorf(string(data)) } + r.Body.Close() } } diff --git a/drivers/alidrive/driver.go b/drivers/alidrive/driver.go index 9ad81566..f695abf6 100644 --- a/drivers/alidrive/driver.go +++ b/drivers/alidrive/driver.go @@ -515,6 +515,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er return err } log.Debugf("%+v", res) + res.Body.Close() //res, err := base.BaseClient.R(). // SetHeader("Content-Type",""). // SetBody(byteData).Put(resp.PartInfoList[i].UploadUrl) diff --git a/drivers/onedrive/onedrive.go b/drivers/onedrive/onedrive.go index 7e4efc71..77bc38ea 100644 --- a/drivers/onedrive/onedrive.go +++ b/drivers/onedrive/onedrive.go @@ -307,8 +307,10 @@ func (driver Onedrive) UploadBig(file *model.FileStream, account *model.Account) res, err := base.HttpClient.Do(req) if res.StatusCode != 201 && res.StatusCode != 202 { data, _ := ioutil.ReadAll(res.Body) + res.Body.Close() return errors.New(string(data)) } + res.Body.Close() } return nil } diff --git a/drivers/yandex/driver.go b/drivers/yandex/driver.go index 3957fec9..5edc5500 100644 --- a/drivers/yandex/driver.go +++ b/drivers/yandex/driver.go @@ -213,7 +213,8 @@ func (driver Yandex) Upload(file *model.FileStream, account *model.Account) erro } req.Header.Set("Content-Length", strconv.FormatUint(file.Size, 10)) req.Header.Set("Content-Type", "application/octet-stream") - _, err = base.HttpClient.Do(req) + res, err := base.HttpClient.Do(req) + res.Body.Close() //res, err := base.RestyClient.R(). // SetHeader("Content-Length", strconv.FormatUint(file.Size, 10)). // SetBody(file).Put(resp.Href) From 732e9eb1c3061f91d130ed0a4005d7bd6a9046d4 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Tue, 10 May 2022 21:40:43 +0800 Subject: [PATCH 07/16] feat:add pprof --- server/static.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server/static.go b/server/static.go index c91ea10a..c60b98d4 100644 --- a/server/static.go +++ b/server/static.go @@ -1,14 +1,16 @@ package server import ( + "io/fs" + "io/ioutil" + "net/http" + "net/http/pprof" + "strings" + "github.com/Xhofe/alist/conf" "github.com/Xhofe/alist/public" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" - "io/fs" - "io/ioutil" - "net/http" - "strings" ) func InitIndex() { @@ -48,6 +50,8 @@ func Static(r *gin.Engine) { c.Status(200) if strings.HasPrefix(c.Request.URL.Path, "/@manage") { _, _ = c.Writer.WriteString(conf.ManageHtml) + } else if strings.HasPrefix(c.Request.URL.Path, "/debug/pprof") { + pprof.Index(c.Writer, c.Request) } else { _, _ = c.Writer.WriteString(conf.IndexHtml) } From 67674835da5f7a9e20154a01261946c616c9de2e Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Tue, 10 May 2022 21:54:44 +0800 Subject: [PATCH 08/16] fix(alidriver):fast upload file is not close --- drivers/alidrive/driver.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/alidrive/driver.go b/drivers/alidrive/driver.go index f695abf6..013d9a49 100644 --- a/drivers/alidrive/driver.go +++ b/drivers/alidrive/driver.go @@ -424,10 +424,17 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er } if account.Bool1 { - buf := make([]byte, 1024) - n, _ := file.Read(buf[:]) - reqBody["pre_hash"] = utils.GetSHA1Encode(string(buf[:n])) - file.File = io.NopCloser(io.MultiReader(bytes.NewReader(buf[:n]), file.File)) + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + io.CopyN(buf, file, 1024) + reqBody["pre_hash"] = utils.GetSHA1Encode(buf.String()) + // 把头部拼接回去 + file.File = struct { + io.Reader + io.Closer + }{ + Reader: io.MultiReader(buf, file.File), + Closer: file.File, + } } else { reqBody["content_hash_name"] = "none" reqBody["proof_version"] = "v1" @@ -454,7 +461,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er return fmt.Errorf("%s", e.Message) } - if e.Code == "PreHashMatched" && account.Bool1 { + if account.Bool1 && e.Code == "PreHashMatched" { tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") if err != nil { return err @@ -499,6 +506,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er return nil } + // 秒传失败 if _, err = tempFile.Seek(0, io.SeekStart); err != nil { return err } From 4b635f06e32a163cb94022fc73ccf52e665cf58b Mon Sep 17 00:00:00 2001 From: Xhofe Date: Wed, 11 May 2022 20:01:15 +0800 Subject: [PATCH 09/16] fix(189pc): delete user-agent for upload --- drivers/189pc/driver.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index 22b6edeb..e8ea2e1c 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -607,6 +607,7 @@ func (driver Cloud189) CommonUpload(file *model.FileStream, parentFile *model.Fi uploadData := uploadUrl.UploadUrls[fmt.Sprint("partNumber_", i)] req, _ := http.NewRequest(http.MethodPut, uploadData.RequestURL, byteData) + req.Header.Del("User-Agent") for k, v := range ParseHttpHeader(uploadData.RequestHeader) { req.Header.Set(k, v) } @@ -717,6 +718,7 @@ func (driver Cloud189) FastUpload(file *model.FileStream, parentFile *model.File for i := 1; i <= count; i++ { uploadData := uploadUrls.UploadUrls[fmt.Sprint("partNumber_", i)] req, _ := http.NewRequest(http.MethodPut, uploadData.RequestURL, io.NewSectionReader(tempFile, int64(i-1)*DEFAULT, DEFAULT)) + req.Header.Del("User-Agent") for k, v := range ParseHttpHeader(uploadData.RequestHeader) { req.Header.Set(k, v) } From 0eef7a129c328efb9dfaf0203ec03bfd38bcd048 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Thu, 12 May 2022 13:26:12 +0800 Subject: [PATCH 10/16] fix(xunlei):the verification code cannot be obtained from the mobile phone number or email --- drivers/xunlei/xunlei.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/xunlei/xunlei.go b/drivers/xunlei/xunlei.go index a1b1a656..4ab1a43f 100644 --- a/drivers/xunlei/xunlei.go +++ b/drivers/xunlei/xunlei.go @@ -117,8 +117,20 @@ func (c *Client) Login(account *model.Account) (err error) { model.SaveAccount(account) }() + meta := make(map[string]string) + if strings.Contains(account.Username, "@") { + meta["email"] = account.Username + } else if len(account.Username) >= 11 { + if !strings.Contains(account.Username, "+") { + account.Username = "+86 " + account.Username + } + meta["phone_number"] = account.Username + } else { + meta["username"] = account.Username + } + url := XLUSER_API_URL + "/auth/signin" - err = c.requestCaptchaToken(getAction(http.MethodPost, url), map[string]string{"username": account.Username}) + err = c.requestCaptchaToken(getAction(http.MethodPost, url), meta) if err != nil { return err } From 041b3587bf35295ba122718777f2120c26626051 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Thu, 12 May 2022 13:27:49 +0800 Subject: [PATCH 11/16] fix(xunlei):turn page --- drivers/xunlei/driver.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/xunlei/driver.go b/drivers/xunlei/driver.go index 2e5b6325..135f3dde 100644 --- a/drivers/xunlei/driver.go +++ b/drivers/xunlei/driver.go @@ -171,14 +171,15 @@ func (driver XunLeiCloud) Files(path string, account *model.Account) ([]model.Fi return nil, err } - time.Sleep(time.Millisecond * 400) + time.Sleep(time.Millisecond * 300) files := make([]model.File, 0) + var pageToken string for { var fileList FileList _, err = GetClient(account).Request("GET", FILE_API_URL, func(r *resty.Request) { r.SetQueryParams(map[string]string{ "parent_id": parentFile.Id, - "page_token": fileList.NextPageToken, + "page_token": pageToken, "with_audit": "true", "filters": `{"phase": {"eq": "PHASE_TYPE_COMPLETE"}, "trashed":{"eq":false}}`, }) @@ -195,6 +196,7 @@ func (driver XunLeiCloud) Files(path string, account *model.Account) ([]model.Fi if fileList.NextPageToken == "" { break } + pageToken = fileList.NextPageToken } if len(files) > 0 { _ = base.SetCache(path, files, account) From 6a67d1cf6970a30c814843d0d09b90fce2933c9a Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Thu, 12 May 2022 19:15:39 +0800 Subject: [PATCH 12/16] fix(xunlei):check captchaToken --- drivers/xunlei/driver.go | 6 ++---- drivers/xunlei/xunlei.go | 11 +++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/xunlei/driver.go b/drivers/xunlei/driver.go index 135f3dde..4eb5b3bf 100644 --- a/drivers/xunlei/driver.go +++ b/drivers/xunlei/driver.go @@ -117,11 +117,9 @@ func (driver XunLeiCloud) Save(account *model.Account, old *model.Account) error client := GetClient(account) // 指定验证通过的captchaToken - if client.captchaToken != "" { - client.Lock() - client.captchaToken = account.CaptchaToken + if account.CaptchaToken != "" { + client.UpdateCaptchaToken(strings.TrimSpace(account.CaptchaToken)) account.CaptchaToken = "" - client.Unlock() } if client.token == "" { diff --git a/drivers/xunlei/xunlei.go b/drivers/xunlei/xunlei.go index 4ab1a43f..2c966c99 100644 --- a/drivers/xunlei/xunlei.go +++ b/drivers/xunlei/xunlei.go @@ -269,3 +269,14 @@ func (c *Client) Request(method string, url string, callback func(*resty.Request } return c.Request(method, url, callback, account) } + +func (c *Client) UpdateCaptchaToken(captchaToken string) bool { + c.Lock() + defer c.Unlock() + + if captchaToken != "" { + c.captchaToken = captchaToken + return true + } + return false +} From f917882a84e2dff1edeaa0ac960c5399d9643404 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Thu, 12 May 2022 19:15:52 +0800 Subject: [PATCH 13/16] perf(xunlei):upload --- drivers/xunlei/driver.go | 60 +++++++++++++++++++++++----------------- go.mod | 3 -- go.sum | 9 ------ 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/drivers/xunlei/driver.go b/drivers/xunlei/driver.go index 4eb5b3bf..fa9db1cc 100644 --- a/drivers/xunlei/driver.go +++ b/drivers/xunlei/driver.go @@ -1,10 +1,6 @@ package xunlei import ( - "fmt" - "io" - "io/ioutil" - "os" "path/filepath" "strconv" "strings" @@ -14,7 +10,10 @@ import ( "github.com/Xhofe/alist/drivers/base" "github.com/Xhofe/alist/model" "github.com/Xhofe/alist/utils" - "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" "github.com/google/uuid" ) @@ -179,6 +178,7 @@ func (driver XunLeiCloud) Files(path string, account *model.Account) ([]model.Fi "parent_id": parentFile.Id, "page_token": pageToken, "with_audit": "true", + "limit": "100", "filters": `{"phase": {"eq": "PHASE_TYPE_COMPLETE"}, "trashed":{"eq":false}}`, }) r.SetResult(&fileList) @@ -357,19 +357,24 @@ func (driver XunLeiCloud) Upload(file *model.FileStream, account *model.Account) return err } - tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") - if err != nil { - return err - } + /* + tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") + if err != nil { + return err + } - defer os.Remove(tempFile.Name()) + defer tempFile.Close() + defer os.Remove(tempFile.Name()) - gcid, err := getGcid(io.TeeReader(file, tempFile), int64(file.Size)) - if err != nil { - return err - } + gcid, err := getGcid(io.TeeReader(file, tempFile), int64(file.Size)) + if err != nil { + return err + } - tempFile.Close() + if _, err = tempFile.Seek(0, io.SeekStart); err != nil { + return err + } + */ var resp UploadTaskResponse _, err = GetClient(account).Request("POST", FILE_API_URL, func(r *resty.Request) { @@ -377,8 +382,8 @@ func (driver XunLeiCloud) Upload(file *model.FileStream, account *model.Account) "kind": FILE, "parent_id": parentFile.Id, "name": file.Name, - "size": fmt.Sprint(file.Size), - "hash": gcid, + "size": file.Size, + "hash": "1CF254FBC456E1B012CD45C546636AA62CF8350E", "upload_type": UPLOAD_TYPE_RESUMABLE, }) r.SetResult(&resp) @@ -390,18 +395,21 @@ func (driver XunLeiCloud) Upload(file *model.FileStream, account *model.Account) param := resp.Resumable.Params if resp.UploadType == UPLOAD_TYPE_RESUMABLE { param.Endpoint = strings.TrimLeft(param.Endpoint, param.Bucket+".") - client, err := oss.New(param.Endpoint, param.AccessKeyID, param.AccessKeySecret, oss.SecurityToken(param.SecurityToken), oss.EnableMD5(true)) - if err != nil { - return err - } - bucket, err := client.Bucket(param.Bucket) - if err != nil { - return err - } - err = bucket.UploadFile(param.Key, tempFile.Name(), 1<<22, oss.Routines(3), oss.Checkpoint(true, ""), oss.Expires(param.Expiration)) + s, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(param.AccessKeyID, param.AccessKeySecret, param.SecurityToken), + Region: aws.String("xunlei"), + Endpoint: aws.String(param.Endpoint), + }) if err != nil { return err } + _, err = s3manager.NewUploader(s).Upload(&s3manager.UploadInput{ + Bucket: aws.String(param.Bucket), + Key: aws.String(param.Key), + Expires: aws.Time(param.Expiration), + Body: file, + }) + return err } return nil } diff --git a/go.mod b/go.mod index 433b2ed0..9da54ec1 100644 --- a/go.mod +++ b/go.mod @@ -27,15 +27,12 @@ require ( require github.com/kr/fs v0.1.0 // indirect require ( - github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/fatih/color v1.13.0 github.com/mattn/go-colorable v0.1.9 // indirect - golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect ) require ( github.com/XiaoMi/pegasus-go-client v0.0.0-20210427083443-f3b6b08bc4c2 // indirect - github.com/aliyun/aliyun-oss-go-sdk v2.2.1+incompatible github.com/beorn7/perks v1.0.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect github.com/cenkalti/backoff/v4 v4.1.0 // indirect diff --git a/go.sum b/go.sum index 7e73c124..7d46f2d7 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/aliyun/aliyun-oss-go-sdk v2.2.1+incompatible h1:uuJIwCFhbZy+zdvLy5zrcIToPEQP0s5CFOZ0Zj03O/w= -github.com/aliyun/aliyun-oss-go-sdk v2.2.1+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/allegro/bigcache/v2 v2.2.5 h1:mRc8r6GQjuJsmSKQNPsR5jQVXc8IJ1xsW5YXUYMLfqI= github.com/allegro/bigcache/v2 v2.2.5/go.mod h1:FppZsIO+IZk7gCuj5FiIDHGygD9xvWQcqg1uIPMb6tY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -34,8 +32,6 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= -github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -466,8 +462,6 @@ github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThC github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= @@ -659,9 +653,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 3b59bb5c090e4456a0b8075c061c4c5546e28cdf Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Thu, 12 May 2022 21:18:46 +0800 Subject: [PATCH 14/16] perf(123):upload --- drivers/123/123.go | 5 +++-- drivers/123/driver.go | 41 ++++++++++++++++------------------------- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/drivers/123/123.go b/drivers/123/123.go index e93f144b..26d87af6 100644 --- a/drivers/123/123.go +++ b/drivers/123/123.go @@ -3,14 +3,15 @@ package _23 import ( "errors" "fmt" + "path/filepath" + "strconv" + "github.com/Xhofe/alist/drivers/base" "github.com/Xhofe/alist/model" "github.com/Xhofe/alist/utils" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" - "path/filepath" - "strconv" ) func (driver Pan123) Login(account *model.Account) error { diff --git a/drivers/123/driver.go b/drivers/123/driver.go index eeb6bebd..1ffa69ef 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -4,6 +4,13 @@ import ( "crypto/md5" "encoding/hex" "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strconv" + "github.com/Xhofe/alist/conf" "github.com/Xhofe/alist/drivers/base" "github.com/Xhofe/alist/model" @@ -13,12 +20,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" log "github.com/sirupsen/logrus" - "io" - "io/ioutil" - "net/url" - "os" - "path/filepath" - "strconv" ) type Pan123 struct{} @@ -300,46 +301,36 @@ func (driver Pan123) Upload(file *model.FileStream, account *model.Account) erro if !parentFile.IsDir() { return base.ErrNotFolder } - parentFileId, _ := strconv.Atoi(parentFile.Id) + tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") if err != nil { return err } - defer func() { - _ = tempFile.Close() - _ = os.Remove(tempFile.Name()) - }() - _, err = io.Copy(tempFile, file) - if err != nil { - return err - } - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err - } + defer tempFile.Close() + defer os.Remove(tempFile.Name()) h := md5.New() - _, err = io.Copy(h, tempFile) - if err != nil { + if _, err = io.Copy(io.MultiWriter(tempFile, h), file); err != nil { return err } etag := hex.EncodeToString(h.Sum(nil)) - log.Debugln("md5:", etag) + _, err = tempFile.Seek(0, io.SeekStart) if err != nil { return err } + data := base.Json{ "driveId": 0, - "duplicate": true, + "duplicate": 2, // 2->覆盖 1->重命名 0->默认 "etag": etag, "fileName": file.GetFileName(), - "parentFileId": parentFileId, + "parentFileId": parentFile.Id, "size": file.GetSize(), "type": 0, } var resp UploadResp _, err = driver.Request("https://www.123pan.com/api/file/upload_request", - base.Post, nil, nil, &data, &resp, false, account) + base.Post, map[string]string{"app-version": "1.1"}, nil, &data, &resp, false, account) //res, err := driver.Post("https://www.123pan.com/api/file/upload_request", data, account) if err != nil { return err From aeb2297f1f247fb1b0020d85dc961be1985ac405 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Thu, 12 May 2022 22:10:24 +0800 Subject: [PATCH 15/16] perf(123):file thumbnail --- drivers/123/123.go | 3 ++- drivers/123/driver.go | 2 +- drivers/123/types.go | 20 +++++++++++--------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/drivers/123/123.go b/drivers/123/123.go index 26d87af6..87f8cafe 100644 --- a/drivers/123/123.go +++ b/drivers/123/123.go @@ -47,6 +47,7 @@ func (driver Pan123) FormatFile(file *File) *model.File { Size: file.Size, Driver: driver.Config().Name, UpdatedAt: file.UpdateAt, + Thumbnail: file.DownloadUrl, } f.Type = file.GetType() return f @@ -66,7 +67,7 @@ func (driver Pan123) GetFiles(parentId string, account *model.Account) ([]File, "parentFileId": parentId, "trashed": "false", } - _, err := driver.Request("https://www.123pan.com/api/file/list", + _, err := driver.Request("https://www.123pan.com/api/file/list/new", base.Get, nil, query, nil, &resp, false, account) if err != nil { return nil, err diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 1ffa69ef..43cf7cf7 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -126,7 +126,7 @@ func (driver Pan123) Files(path string, account *model.Account) ([]model.File, e _ = base.SetCache(path, rawFiles, account) } } - files := make([]model.File, 0) + files := make([]model.File, 0, len(rawFiles)) for _, file := range rawFiles { files = append(files, *driver.FormatFile(&file)) } diff --git a/drivers/123/types.go b/drivers/123/types.go index 012d350e..c464628c 100644 --- a/drivers/123/types.go +++ b/drivers/123/types.go @@ -1,20 +1,22 @@ package _23 import ( - "github.com/Xhofe/alist/conf" - "github.com/Xhofe/alist/utils" "path" "time" + + "github.com/Xhofe/alist/conf" + "github.com/Xhofe/alist/utils" ) type File struct { - FileName string `json:"FileName"` - Size int64 `json:"Size"` - UpdateAt *time.Time `json:"UpdateAt"` - FileId int64 `json:"FileId"` - Type int `json:"Type"` - Etag string `json:"Etag"` - S3KeyFlag string `json:"S3KeyFlag"` + FileName string `json:"FileName"` + Size int64 `json:"Size"` + UpdateAt *time.Time `json:"UpdateAt"` + FileId int64 `json:"FileId"` + Type int `json:"Type"` + Etag string `json:"Etag"` + S3KeyFlag string `json:"S3KeyFlag"` + DownloadUrl string `json:"DownloadUrl"` } func (f File) GetSize() uint64 { From 87e339850d760314827f932b1c18f27ee9539ee6 Mon Sep 17 00:00:00 2001 From: Xhofe Date: Fri, 13 May 2022 17:38:22 +0800 Subject: [PATCH 16/16] fix(ftp): remove dir (#1082) --- drivers/ftp/driver.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/ftp/driver.go b/drivers/ftp/driver.go index d5915a95..53f1a4e7 100644 --- a/drivers/ftp/driver.go +++ b/drivers/ftp/driver.go @@ -227,13 +227,21 @@ func (driver FTP) Copy(src string, dst string, account *model.Account) error { func (driver FTP) Delete(path string, account *model.Account) error { path = utils.ParsePath(path) + file, err := driver.File(path, account) + if err != nil { + return err + } realPath := utils.Join(account.RootFolder, path) conn, err := driver.Login(account) if err != nil { return err } //defer func() { _ = conn.Quit() }() - err = conn.Delete(realPath) + if file.IsDir() { + err = conn.RemoveDirRecur(realPath) + } else { + err = conn.Delete(realPath) + } return err }