From bf0ee3d31501a2963ebaffbb90b6cb50bdaf7bae Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Mon, 9 May 2022 12:43:51 +0800 Subject: [PATCH 1/6] refactor(baidu.photo): add a file api of download --- drivers/baiduphoto/driver.go | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/drivers/baiduphoto/driver.go b/drivers/baiduphoto/driver.go index 99010ac1..ce871019 100644 --- a/drivers/baiduphoto/driver.go +++ b/drivers/baiduphoto/driver.go @@ -43,6 +43,14 @@ func (driver Baidu) Items() []base.Item { Label: "album_id", Type: base.TypeString, }, + { + Name: "internal_type", + Label: "download api", + Type: base.TypeSelect, + Required: true, + Values: "file,album", + Default: "album", + }, { Name: "client_id", Label: "client id", @@ -156,6 +164,13 @@ func (driver Baidu) Files(path string, account *model.Account) ([]model.File, er } func (driver Baidu) Link(args base.Args, account *model.Account) (*base.Link, error) { + if account.InternalType == "file" { + return driver.LinkFile(args, account) + } + return driver.LinkAlbum(args, account) +} + +func (driver Baidu) LinkAlbum(args base.Args, account *model.Account) (*base.Link, error) { file, err := driver.File(args.Path, account) if err != nil { return nil, err @@ -190,6 +205,43 @@ func (driver Baidu) Link(args base.Args, account *model.Account) (*base.Link, er }, nil } +func (driver Baidu) LinkFile(args base.Args, account *model.Account) (*base.Link, error) { + file, err := driver.File(args.Path, account) + if err != nil { + return nil, err + } + + if !IsAlbumFile(file) { + return nil, base.ErrNotSupport + } + + album, err := driver.File(utils.Dir(utils.ParsePath(args.Path)), account) + if err != nil { + return nil, err + } + // 拷贝到根目录 + cfile, err := driver.CopyAlbumFile(album.Id, account, file.Id) + if err != nil { + return nil, err + } + // 获取文件下载地址 + res, err := base.NoRedirectClient.R(). + SetQueryParams(map[string]string{ + "access_token": account.AccessToken, + "fsid": fmt.Sprint(cfile.Fsid), + }). + Head(FILE_API_URL + "/download") + if err != nil { + return nil, err + } + return &base.Link{ + Headers: []base.Header{ + {Name: "User-Agent", Value: base.UserAgent}, + }, + Url: res.Header().Get("location"), + }, nil +} + func (driver Baidu) Path(path string, account *model.Account) (*model.File, []model.File, error) { path = utils.ParsePath(path) file, err := driver.File(path, account) From 52dcbfe1a4bfc2e772530328226011a616acb044 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Mon, 9 May 2022 18:14:52 +0800 Subject: [PATCH 2/6] fix(xunlei):missing x-client-id error in some user requests --- drivers/xunlei/driver.go | 4 +++- drivers/xunlei/xunlei.go | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/xunlei/driver.go b/drivers/xunlei/driver.go index bd2dddbc..2e5b6325 100644 --- a/drivers/xunlei/driver.go +++ b/drivers/xunlei/driver.go @@ -105,7 +105,7 @@ func (driver XunLeiCloud) Items() []base.Item { Label: "device id", Default: utils.GetMD5Encode(uuid.NewString()), Type: base.TypeString, - Required: false, + Required: true, }, } } @@ -118,8 +118,10 @@ func (driver XunLeiCloud) Save(account *model.Account, old *model.Account) error client := GetClient(account) // 指定验证通过的captchaToken if client.captchaToken != "" { + client.Lock() client.captchaToken = account.CaptchaToken account.CaptchaToken = "" + client.Unlock() } if client.token == "" { diff --git a/drivers/xunlei/xunlei.go b/drivers/xunlei/xunlei.go index 13acd202..a1b1a656 100644 --- a/drivers/xunlei/xunlei.go +++ b/drivers/xunlei/xunlei.go @@ -73,6 +73,8 @@ func (c *Client) requestCaptchaToken(action string, meta map[string]string) erro SetBody(¶m). SetError(&e). SetResult(&resp). + SetHeader("X-Device-Id", c.deviceID). + SetQueryParam("client_id", c.clientID). Post(XLUSER_API_URL + "/shield/captcha/init") if err != nil { return err @@ -133,6 +135,8 @@ func (c *Client) Login(account *model.Account) (err error) { Username: account.Username, Password: account.Password, }). + SetHeader("X-Device-Id", c.deviceID). + SetQueryParam("client_id", c.clientID). Post(url) if err != nil { return err @@ -184,6 +188,8 @@ func (c *Client) RefreshToken() error { "client_id": c.clientID, "client_secret": c.clientSecret, }). + SetHeader("X-Device-Id", c.deviceID). + SetQueryParam("client_id", c.clientID). Post(XLUSER_API_URL + "/auth/token") if err != nil { return err @@ -211,7 +217,8 @@ func (c *Client) Request(method string, url string, callback func(*resty.Request "X-Captcha-Token": c.captchaToken, "User-Agent": c.userAgent, "client_id": c.clientID, - }).SetQueryParam("client_id", c.clientID) + }). + SetQueryParam("client_id", c.clientID) if callback != nil { callback(req) } From a9027c0f06bd1610c40f2b175fd5c688f42f8191 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Tue, 10 May 2022 11:43:54 +0800 Subject: [PATCH 3/6] fix(baidu.photo):update download api --- drivers/baiduphoto/baidu.go | 2 +- drivers/baiduphoto/driver.go | 19 +++++++++---------- drivers/baiduphoto/types.go | 2 +- drivers/baiduphoto/util.go | 7 ++++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/baiduphoto/baidu.go b/drivers/baiduphoto/baidu.go index 287ec433..c192939b 100644 --- a/drivers/baiduphoto/baidu.go +++ b/drivers/baiduphoto/baidu.go @@ -87,7 +87,7 @@ func (driver Baidu) GetAllFile(account *model.Account) (files []File, err error) for { var resp FileListResp - _, err = driver.Request(http.MethodGet, FILE_API_URL+"/list", func(r *resty.Request) { + _, err = driver.Request(http.MethodGet, FILE_API_URL_V1+"/list", func(r *resty.Request) { r.SetQueryParams(map[string]string{ "need_thumbnail": "1", "need_filter_hidden": "0", diff --git a/drivers/baiduphoto/driver.go b/drivers/baiduphoto/driver.go index ce871019..833ef89f 100644 --- a/drivers/baiduphoto/driver.go +++ b/drivers/baiduphoto/driver.go @@ -224,13 +224,12 @@ func (driver Baidu) LinkFile(args base.Args, account *model.Account) (*base.Link if err != nil { return nil, err } - // 获取文件下载地址 - res, err := base.NoRedirectClient.R(). - SetQueryParams(map[string]string{ - "access_token": account.AccessToken, - "fsid": fmt.Sprint(cfile.Fsid), - }). - Head(FILE_API_URL + "/download") + + res, err := driver.Request(http.MethodGet, FILE_API_URL_V2+"/download", func(r *resty.Request) { + r.SetQueryParams(map[string]string{ + "fsid": fmt.Sprint(cfile.Fsid), + }) + }, account) if err != nil { return nil, err } @@ -238,7 +237,7 @@ func (driver Baidu) LinkFile(args base.Args, account *model.Account) (*base.Link Headers: []base.Header{ {Name: "User-Agent", Value: base.UserAgent}, }, - Url: res.Header().Get("location"), + Url: utils.Json.Get(res.Body(), "dlink").ToString(), }, nil } @@ -454,7 +453,7 @@ func (driver Baidu) Upload(file *model.FileStream, account *model.Account) error // 预上传 var precreateResp PrecreateResp - _, err = driver.Request(http.MethodPost, FILE_API_URL+"/precreate", func(r *resty.Request) { + _, err = driver.Request(http.MethodPost, FILE_API_URL_V1+"/precreate", func(r *resty.Request) { r.SetFormData(params) r.SetResult(&precreateResp) }, account) @@ -483,7 +482,7 @@ func (driver Baidu) Upload(file *model.FileStream, account *model.Account) error fallthrough case 2: // 创建文件 params["uploadid"] = precreateResp.UploadID - _, err = driver.Request(http.MethodPost, FILE_API_URL+"/create", func(r *resty.Request) { + _, err = driver.Request(http.MethodPost, FILE_API_URL_V1+"/create", func(r *resty.Request) { r.SetFormData(params) r.SetResult(&precreateResp) }, account) diff --git a/drivers/baiduphoto/types.go b/drivers/baiduphoto/types.go index 734b5a53..926329d6 100644 --- a/drivers/baiduphoto/types.go +++ b/drivers/baiduphoto/types.go @@ -97,7 +97,7 @@ type ( type ( UploadFile struct { FsID int64 `json:"fs_id"` - Size int `json:"size"` + Size int64 `json:"size"` Md5 string `json:"md5"` ServerFilename string `json:"server_filename"` Path string `json:"path"` diff --git a/drivers/baiduphoto/util.go b/drivers/baiduphoto/util.go index 5ab40a52..2c0542c7 100644 --- a/drivers/baiduphoto/util.go +++ b/drivers/baiduphoto/util.go @@ -13,9 +13,10 @@ import ( ) const ( - API_URL = "https://photo.baidu.com/youai" - ALBUM_API_URL = API_URL + "/album/v1" - FILE_API_URL = API_URL + "/file/v1" + API_URL = "https://photo.baidu.com/youai" + ALBUM_API_URL = API_URL + "/album/v1" + FILE_API_URL_V1 = API_URL + "/file/v1" + FILE_API_URL_V2 = API_URL + "/file/v2" ) var ( From b6af9aa587a6305dd2eeb1847a7d398c6e579464 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Tue, 10 May 2022 21:35:42 +0800 Subject: [PATCH 4/6] fix(139,189,189pc,alidrive,onedrive,yandex):http response body is not close #1072 --- drivers/139/driver.go | 1 + drivers/189/189.go | 1 + drivers/189pc/driver.go | 4 ++++ drivers/alidrive/driver.go | 1 + drivers/onedrive/onedrive.go | 2 ++ drivers/yandex/driver.go | 3 ++- 6 files changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index b0025fcc..8864787c 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -448,6 +448,7 @@ func (driver Cloud139) Upload(file *model.FileStream, account *model.Account) er return err } log.Debugf("%+v", res) + res.Body.Close() start += byteSize } return nil diff --git a/drivers/189/189.go b/drivers/189/189.go index f921a749..ac6ea189 100644 --- a/drivers/189/189.go +++ b/drivers/189/189.go @@ -577,6 +577,7 @@ func (driver Cloud189) NewUpload(file *model.FileStream, account *model.Account) r, err := base.HttpClient.Do(req) log.Debugf("%+v %+v", r, r.Request.Header) + r.Body.Close() if err != nil { return err } diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index b6ca7a1b..22b6edeb 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -619,8 +619,10 @@ func (driver Cloud189) CommonUpload(file *model.FileStream, parentFile *model.Fi } if r.StatusCode != http.StatusOK { data, _ := io.ReadAll(r.Body) + r.Body.Close() return fmt.Errorf(string(data)) } + r.Body.Close() } fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil))) @@ -727,8 +729,10 @@ func (driver Cloud189) FastUpload(file *model.FileStream, parentFile *model.File } if r.StatusCode != http.StatusOK { data, _ := io.ReadAll(r.Body) + r.Body.Close() return fmt.Errorf(string(data)) } + r.Body.Close() } } diff --git a/drivers/alidrive/driver.go b/drivers/alidrive/driver.go index 9ad81566..f695abf6 100644 --- a/drivers/alidrive/driver.go +++ b/drivers/alidrive/driver.go @@ -515,6 +515,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er return err } log.Debugf("%+v", res) + res.Body.Close() //res, err := base.BaseClient.R(). // SetHeader("Content-Type",""). // SetBody(byteData).Put(resp.PartInfoList[i].UploadUrl) diff --git a/drivers/onedrive/onedrive.go b/drivers/onedrive/onedrive.go index 7e4efc71..77bc38ea 100644 --- a/drivers/onedrive/onedrive.go +++ b/drivers/onedrive/onedrive.go @@ -307,8 +307,10 @@ func (driver Onedrive) UploadBig(file *model.FileStream, account *model.Account) res, err := base.HttpClient.Do(req) if res.StatusCode != 201 && res.StatusCode != 202 { data, _ := ioutil.ReadAll(res.Body) + res.Body.Close() return errors.New(string(data)) } + res.Body.Close() } return nil } diff --git a/drivers/yandex/driver.go b/drivers/yandex/driver.go index 3957fec9..5edc5500 100644 --- a/drivers/yandex/driver.go +++ b/drivers/yandex/driver.go @@ -213,7 +213,8 @@ func (driver Yandex) Upload(file *model.FileStream, account *model.Account) erro } req.Header.Set("Content-Length", strconv.FormatUint(file.Size, 10)) req.Header.Set("Content-Type", "application/octet-stream") - _, err = base.HttpClient.Do(req) + res, err := base.HttpClient.Do(req) + res.Body.Close() //res, err := base.RestyClient.R(). // SetHeader("Content-Length", strconv.FormatUint(file.Size, 10)). // SetBody(file).Put(resp.Href) From 732e9eb1c3061f91d130ed0a4005d7bd6a9046d4 Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Tue, 10 May 2022 21:40:43 +0800 Subject: [PATCH 5/6] feat:add pprof --- server/static.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server/static.go b/server/static.go index c91ea10a..c60b98d4 100644 --- a/server/static.go +++ b/server/static.go @@ -1,14 +1,16 @@ package server import ( + "io/fs" + "io/ioutil" + "net/http" + "net/http/pprof" + "strings" + "github.com/Xhofe/alist/conf" "github.com/Xhofe/alist/public" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" - "io/fs" - "io/ioutil" - "net/http" - "strings" ) func InitIndex() { @@ -48,6 +50,8 @@ func Static(r *gin.Engine) { c.Status(200) if strings.HasPrefix(c.Request.URL.Path, "/@manage") { _, _ = c.Writer.WriteString(conf.ManageHtml) + } else if strings.HasPrefix(c.Request.URL.Path, "/debug/pprof") { + pprof.Index(c.Writer, c.Request) } else { _, _ = c.Writer.WriteString(conf.IndexHtml) } From 67674835da5f7a9e20154a01261946c616c9de2e Mon Sep 17 00:00:00 2001 From: foxxorcat Date: Tue, 10 May 2022 21:54:44 +0800 Subject: [PATCH 6/6] fix(alidriver):fast upload file is not close --- drivers/alidrive/driver.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/alidrive/driver.go b/drivers/alidrive/driver.go index f695abf6..013d9a49 100644 --- a/drivers/alidrive/driver.go +++ b/drivers/alidrive/driver.go @@ -424,10 +424,17 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er } if account.Bool1 { - buf := make([]byte, 1024) - n, _ := file.Read(buf[:]) - reqBody["pre_hash"] = utils.GetSHA1Encode(string(buf[:n])) - file.File = io.NopCloser(io.MultiReader(bytes.NewReader(buf[:n]), file.File)) + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + io.CopyN(buf, file, 1024) + reqBody["pre_hash"] = utils.GetSHA1Encode(buf.String()) + // 把头部拼接回去 + file.File = struct { + io.Reader + io.Closer + }{ + Reader: io.MultiReader(buf, file.File), + Closer: file.File, + } } else { reqBody["content_hash_name"] = "none" reqBody["proof_version"] = "v1" @@ -454,7 +461,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er return fmt.Errorf("%s", e.Message) } - if e.Code == "PreHashMatched" && account.Bool1 { + if account.Bool1 && e.Code == "PreHashMatched" { tempFile, err := ioutil.TempFile(conf.Conf.TempDir, "file-*") if err != nil { return err @@ -499,6 +506,7 @@ func (driver AliDrive) Upload(file *model.FileStream, account *model.Account) er return nil } + // 秒传失败 if _, err = tempFile.Seek(0, io.SeekStart); err != nil { return err }