From cfee536b96f38e5ba3f3575fab4e89f6c0e1bc5b Mon Sep 17 00:00:00 2001 From: wenmig Date: Fri, 23 Jun 2023 17:36:40 +0800 Subject: [PATCH] feat: add Dropbox driver (#4639 close #4590) Co-authored-by: Andy Hsu --- README.md | 1 + README_cn.md | 1 + drivers/all.go | 1 + drivers/dropbox/driver.go | 224 ++++++++++++++++++++++++++++++++++++++ drivers/dropbox/meta.go | 42 +++++++ drivers/dropbox/types.go | 79 ++++++++++++++ drivers/dropbox/util.go | 199 +++++++++++++++++++++++++++++++++ pkg/utils/slice.go | 9 ++ 8 files changed, 556 insertions(+) create mode 100644 drivers/dropbox/driver.go create mode 100644 drivers/dropbox/meta.go create mode 100644 drivers/dropbox/types.go create mode 100644 drivers/dropbox/util.go diff --git a/README.md b/README.md index 020c6b40..c96cc33b 100755 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ English | [中文](./README_cn.md) | [Contributing](./CONTRIBUTING.md) | [CODE_O - [x] SMB - [x] [115](https://115.com/) - [X] Cloudreve + - [x] [Dropbox](https://www.dropbox.com/) - [x] Easy to deploy and out-of-the-box - [x] File preview (PDF, markdown, code, plain text, ...) - [x] Image preview in gallery mode diff --git a/README_cn.md b/README_cn.md index e796a11a..fdc8d096 100644 --- a/README_cn.md +++ b/README_cn.md @@ -72,6 +72,7 @@ - [x] SMB - [x] [115](https://115.com/) - [X] Cloudreve + - [x] [Dropbox](https://www.dropbox.com/) - [x] 部署方便，开箱即用 - [x] 文件预览（PDF、markdown、代码、纯文本……） - [x] 画廊模式下的图像预览 diff --git a/drivers/all.go b/drivers/all.go index d9b0b362..314cf9cb 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -16,6 +16,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/baidu_photo" _ "github.com/alist-org/alist/v3/drivers/baidu_share" _ "github.com/alist-org/alist/v3/drivers/cloudreve" + _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/ftp" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" diff --git a/drivers/dropbox/driver.go b/drivers/dropbox/driver.go new file mode 100644 index 00000000..b98d32f9 --- /dev/null +++ b/drivers/dropbox/driver.go @@ -0,0 +1,224 @@ +package dropbox + +import ( + "context" + "fmt" + "io" + "math" + "net/http" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +type Dropbox struct { + model.Storage + Addition + base string + contentBase string +} + +func (d *Dropbox) Config() driver.Config { + return config +} + +func (d *Dropbox) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Dropbox) Init(ctx context.Context) error { + query := "foo" + res, err := d.request("/2/check/user", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "query": query, + }) + }) + if err != nil { + return err + } + result := utils.Json.Get(res, "result").ToString() + if result != query { + return fmt.Errorf("failed to check user: %s", string(res)) + } + return nil +} + +func (d *Dropbox) Drop(ctx context.Context) error { + return nil +} + +func (d *Dropbox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(ctx, dir.GetPath()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *Dropbox) link(ctx context.Context, file model.Obj) (*model.Link, error) { + res, err := d.request("/2/files/get_temporary_link", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "path": file.GetID(), + }) + }) + if err != nil { + return nil, err + } + url := utils.Json.Get(res, "link").ToString() + exp := time.Hour + return &model.Link{ + URL: url, + Expiration: &exp, + }, nil +} + +func (d *Dropbox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + return d.link(ctx, file) +} + +func (d *Dropbox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + _, err := d.request("/2/files/create_folder_v2", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "autorename": false, + "path": parentDir.GetPath() + "/" + dirName, + }) + }) + return err +} + +func (d *Dropbox) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + toPath := dstDir.GetPath() + "/" + srcObj.GetName() + + _, err := d.request("/2/files/move_v2", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "allow_ownership_transfer": false, + "allow_shared_folder": false, + "autorename": false, + "from_path": srcObj.GetID(), + "to_path": toPath, + }) + }) + return err +} + +func (d *Dropbox) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + path := srcObj.GetPath() + fileName := srcObj.GetName() + toPath := path[:len(path)-len(fileName)] + newName + + _, err := d.request("/2/files/move_v2", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "allow_ownership_transfer": false, + "allow_shared_folder": false, + "autorename": false, + "from_path": srcObj.GetID(), + "to_path": toPath, + }) + }) + return err +} + +func (d *Dropbox) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + toPath := dstDir.GetPath() + "/" + srcObj.GetName() + _, err := d.request("/2/files/copy_v2", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "allow_ownership_transfer": false, + "allow_shared_folder": false, + "autorename": false, + "from_path": srcObj.GetID(), + "to_path": toPath, + }) + }) + return err +} + +func (d *Dropbox) Remove(ctx context.Context, obj model.Obj) error { + uri := "/2/files/delete_v2" + _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "path": obj.GetID(), + }) + }) + return err +} + +func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + // 1. start + sessionId, err := d.startUploadSession(ctx) + if err != nil { + return err + } + + // 2.append + // A single request should not upload more than 150 MB, and each call must be multiple of 4MB (except for last call) + const PartSize = 20971520 + count := 1 + if stream.GetSize() > PartSize { + count = int(math.Ceil(float64(stream.GetSize()) / float64(PartSize))) + } + offset := int64(0) + + for i := 0; i < count; i++ { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + + start := i * PartSize + byteSize := stream.GetSize() - int64(start) + if byteSize > PartSize { + byteSize = PartSize + } + + url := d.contentBase + "/2/files/upload_session/append_v2" + reader := io.LimitReader(stream, PartSize) + req, err := http.NewRequest(http.MethodPost, url, reader) + if err != nil { + log.Errorf("failed to update file when append to upload session, err: %+v", err) + return err + } + req = req.WithContext(ctx) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Authorization", "Bearer "+d.AccessToken) + + args := UploadAppendArgs{ + Close: false, + Cursor: UploadCursor{ + Offset: offset, + SessionID: sessionId, + }, + } + argsJson, _ := utils.Json.MarshalToString(args) + req.Header.Set("Dropbox-API-Arg", argsJson) + + res, err := base.HttpClient.Do(req) + _ = res.Body.Close() + + if err != nil { + return err + } + + if count > 0 { + up((i + 1) * 100 / count) + } + + offset += byteSize + + } + // 3.finish + toPath := dstDir.GetPath() + "/" + stream.GetName() + err2 := d.finishUploadSession(ctx, toPath, offset, sessionId) + if err2 != nil { + return err2 + } + + return err +} + +var _ driver.Driver = (*Dropbox)(nil) diff --git a/drivers/dropbox/meta.go b/drivers/dropbox/meta.go new file mode 100644 index 00000000..07ab4469 --- /dev/null +++ b/drivers/dropbox/meta.go @@ -0,0 +1,42 @@ +package dropbox + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +const ( + DefaultClientID = "76lrwrklhdn1icb" +) + +type Addition struct { + RefreshToken string `json:"refresh_token" required:"true"` + driver.RootPath + + OauthTokenURL string `json:"oauth_token_url" default:"https://api.xhofe.top/alist/dropbox/token"` + ClientID string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"` + ClientSecret string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"` + + AccessToken string +} + +var config = driver.Config{ + Name: "Dropbox", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Dropbox{ + base: "https://api.dropboxapi.com", + contentBase: "https://content.dropboxapi.com", + } + }) +} diff --git a/drivers/dropbox/types.go b/drivers/dropbox/types.go new file mode 100644 index 00000000..7ddbb968 --- /dev/null +++ b/drivers/dropbox/types.go @@ -0,0 +1,79 @@ +package dropbox + +import ( + "github.com/alist-org/alist/v3/internal/model" + "time" +) + +type TokenResp struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + ExpiresIn int `json:"expires_in"` +} + +type ErrorResp struct { + Error struct { + Tag string `json:".tag"` + } `json:"error"` + ErrorSummary string `json:"error_summary"` +} + +type RefreshTokenErrorResp struct { + Error string `json:"error"` + ErrorDescription string `json:"error_description"` +} + +type File struct { + Tag string `json:".tag"` + Name string `json:"name"` + PathLower string `json:"path_lower"` + PathDisplay string `json:"path_display"` + ID string `json:"id"` + ClientModified time.Time `json:"client_modified"` + ServerModified time.Time `json:"server_modified"` + Rev string `json:"rev"` + Size int `json:"size"` + IsDownloadable bool `json:"is_downloadable"` + ContentHash string `json:"content_hash"` +} + +type ListResp struct { + Entries []File `json:"entries"` + Cursor string `json:"cursor"` + HasMore bool `json:"has_more"` +} + +type UploadCursor struct { + Offset int64 `json:"offset"` + SessionID string `json:"session_id"` +} + +type UploadAppendArgs struct { + Close bool `json:"close"` + Cursor UploadCursor `json:"cursor"` +} + +type UploadFinishArgs struct { + Commit struct { + Autorename bool `json:"autorename"` + Mode string `json:"mode"` + Mute bool `json:"mute"` + Path string `json:"path"` + StrictConflict bool `json:"strict_conflict"` + } `json:"commit"` + Cursor UploadCursor `json:"cursor"` +} + +func fileToObj(f File) *model.ObjThumb { + return &model.ObjThumb{ + Object: model.Object{ + ID: f.ID, + Path: f.PathDisplay, + Name: f.Name, + Size: int64(f.Size), + Modified: f.ServerModified, + IsFolder: f.Tag == "folder", + }, + Thumbnail: model.Thumbnail{}, + } +} diff --git a/drivers/dropbox/util.go b/drivers/dropbox/util.go new file mode 100644 index 00000000..891ca1b7 --- /dev/null +++ b/drivers/dropbox/util.go @@ -0,0 +1,199 @@ +package dropbox + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +func (d *Dropbox) refreshToken() error { + url := d.base + "/oauth2/token" + if utils.SliceContains([]string{"", DefaultClientID}, d.ClientID) { + url = d.OauthTokenURL + } + var tokenResp TokenResp + resp, err := base.RestyClient.R(). + //ForceContentType("application/x-www-form-urlencoded"). + //SetBasicAuth(d.ClientID, d.ClientSecret). + SetFormData(map[string]string{ + "grant_type": "refresh_token", + "refresh_token": d.RefreshToken, + "client_id": d.ClientID, + "client_secret": d.ClientSecret, + }). + Post(url) + if err != nil { + return err + } + log.Debugf("[dropbox] refresh token response: %s", resp.String()) + if resp.StatusCode() != 200 { + return fmt.Errorf("failed to refresh token: %s", resp.String()) + } + _ = utils.Json.UnmarshalFromString(resp.String(), &tokenResp) + d.AccessToken = tokenResp.AccessToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Dropbox) request(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeader("Authorization", "Bearer "+d.AccessToken) + if method == http.MethodPost { + req.SetHeader("Content-Type", "application/json") + } + if callback != nil { + callback(req) + } + var e ErrorResp + req.SetError(&e) + res, err := req.Execute(method, d.base+uri) + if err != nil { + return nil, err + } + log.Debugf("[dropbox] request (%s) response: %s", uri, res.String()) + isRetry := len(retry) > 0 && retry[0] + if res.StatusCode() != 200 { + body := res.String() + if !isRetry && (utils.SliceMeet([]string{"expired_access_token", "invalid_access_token", "authorization"}, body, + func(item string, v string) bool { + return strings.Contains(v, item) + }) || d.AccessToken == "") { + err = d.refreshToken() + if err != nil { + return nil, err + } + return d.request(uri, method, callback, true) + } + return nil, fmt.Errorf("%s:%s", e.Error, e.ErrorSummary) + } + return res.Body(), nil +} + +func (d *Dropbox) list(ctx context.Context, data base.Json, isContinue bool) (*ListResp, error) { + var resp ListResp + uri := "/2/files/list_folder" + if isContinue { + uri += "/continue" + } + _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { + req.SetBody(data).SetResult(&resp) + }) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Dropbox) getFiles(ctx context.Context, path string) ([]File, error) { + hasMore := true + var marker string + res := make([]File, 0) + + data := base.Json{ + "include_deleted": false, + "include_has_explicit_shared_members": false, + "include_mounted_folders": false, + "include_non_downloadable_files": false, + "limit": 2000, + "path": path, + "recursive": false, + } + resp, err := d.list(ctx, data, false) + if err != nil { + return nil, err + } + marker = resp.Cursor + hasMore = resp.HasMore + res = append(res, resp.Entries...) + + for hasMore { + data := base.Json{ + "cursor": marker, + } + resp, err := d.list(ctx, data, true) + if err != nil { + return nil, err + } + marker = resp.Cursor + hasMore = resp.HasMore + res = append(res, resp.Entries...) + } + return res, nil +} + +func (d *Dropbox) finishUploadSession(ctx context.Context, toPath string, offset int64, sessionId string) error { + url := d.contentBase + "/2/files/upload_session/finish" + req, err := http.NewRequest(http.MethodPost, url, nil) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Authorization", "Bearer "+d.AccessToken) + + uploadFinishArgs := UploadFinishArgs{ + Commit: struct { + Autorename bool `json:"autorename"` + Mode string `json:"mode"` + Mute bool `json:"mute"` + Path string `json:"path"` + StrictConflict bool `json:"strict_conflict"` + }{ + Autorename: true, + Mode: "add", + Mute: false, + Path: toPath, + StrictConflict: false, + }, + Cursor: UploadCursor{ + Offset: offset, + SessionID: sessionId, + }, + } + + argsJson, err := utils.Json.MarshalToString(uploadFinishArgs) + if err != nil { + return err + } + req.Header.Set("Dropbox-API-Arg", argsJson) + + res, err := base.HttpClient.Do(req) + if err != nil { + log.Errorf("failed to update file when finish session, err: %+v", err) + return err + } + _ = res.Body.Close() + return nil +} + +func (d *Dropbox) startUploadSession(ctx context.Context) (string, error) { + url := d.contentBase + "/2/files/upload_session/start" + req, err := http.NewRequest(http.MethodPost, url, nil) + if err != nil { + return "", err + } + req = req.WithContext(ctx) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Authorization", "Bearer "+d.AccessToken) + req.Header.Set("Dropbox-API-Arg", "{\"close\":false}") + + res, err := base.HttpClient.Do(req) + if err != nil { + log.Errorf("failed to update file when start session, err: %+v", err) + return "", err + } + + body, err := io.ReadAll(res.Body) + sessionId := utils.Json.Get(body, "session_id").ToString() + + _ = res.Body.Close() + return sessionId, nil +} diff --git a/pkg/utils/slice.go b/pkg/utils/slice.go index 8d2aa2a2..d9adaeed 100644 --- a/pkg/utils/slice.go +++ b/pkg/utils/slice.go @@ -60,3 +60,12 @@ func MergeErrors(errs ...error) error { } return nil } + +func SliceMeet[T1, T2 any](arr []T1, v T2, meet func(item T1, v T2) bool) bool { + for _, item := range arr { + if meet(item, v) { + return true + } + } + return false +}