From 09ef7c7106419b36b12048cfe62ac4b48371c8f5 Mon Sep 17 00:00:00 2001 From: Noah Hsu Date: Wed, 15 Jun 2022 20:31:23 +0800 Subject: [PATCH] refactor: change driver interface --- conf/config.go | 28 ++--- drivers/local/driver.go | 64 +++++----- internal/driver/driver.go | 22 ++-- internal/fs/fs.go | 5 + internal/fs/read.go | 4 +- internal/fs/util.go | 6 +- internal/model/file.go | 6 +- internal/model/{ifile.go => object.go} | 5 +- internal/model/stream.go | 2 +- internal/operations/account.go | 4 +- internal/operations/fs.go | 161 +++++++++++++++---------- internal/operations/path.go | 10 +- 12 files changed, 181 insertions(+), 136 deletions(-) create mode 100644 internal/fs/fs.go rename internal/model/{ifile.go => object.go} (85%) diff --git a/conf/config.go b/conf/config.go index cd276c31..730896a9 100644 --- a/conf/config.go +++ b/conf/config.go @@ -18,11 +18,6 @@ type Scheme struct { KeyFile string `json:"key_file" env:"KEY_FILE"` } -type CacheConfig struct { - Expiration int64 `json:"expiration" env:"CACHE_EXPIRATION"` - CleanupInterval int64 `json:"cleanup_interval" env:"CLEANUP_INTERVAL"` -} - type LogConfig struct { Enable bool `json:"enable" env:"log_enable"` Path string `json:"path" env:"LOG_PATH"` @@ -32,15 +27,15 @@ type LogConfig struct { } type Config struct { - Force bool `json:"force"` - Address string `json:"address" env:"ADDR"` - Port int `json:"port" env:"PORT"` - Assets string `json:"assets" env:"ASSETS"` - Database Database `json:"database"` - Scheme Scheme `json:"scheme"` - Cache CacheConfig `json:"cache"` - TempDir string `json:"temp_dir" env:"TEMP_DIR"` - Log LogConfig `json:"log"` + Force bool `json:"force"` + Address string `json:"address" env:"ADDR"` + Port int `json:"port" env:"PORT"` + CaCheExpiration int `json:"cache_expiration" env:"CACHE_EXPIRATION"` + Assets string `json:"assets" env:"ASSETS"` + Database Database `json:"database"` + Scheme Scheme `json:"scheme"` + TempDir string `json:"temp_dir" env:"TEMP_DIR"` + Log LogConfig `json:"log"` } func DefaultConfig() *Config { @@ -55,10 +50,7 @@ func DefaultConfig() *Config { TablePrefix: "x_", DBFile: "data/data.db", }, - Cache: CacheConfig{ - Expiration: 60, - CleanupInterval: 120, - }, + CaCheExpiration: 30, Log: LogConfig{ Enable: true, Path: "log/%Y-%m-%d-%H:%M.log", diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 545dccac..e4b007a3 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -44,12 +44,42 @@ func (d *Driver) GetAddition() driver.Additional { return d.Addition } -func (d *Driver) List(ctx context.Context, path string) ([]model.FileInfo, error) { +func (d *Driver) List(ctx context.Context, dir model.Object) ([]model.Object, error) { //TODO implement me panic("implement me") } -func (d *Driver) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, error) { +func (d *Driver) Link(ctx context.Context, file model.Object, args model.LinkArgs) (*model.Link, error) { + //TODO implement me + panic("implement me") +} + +func (d *Driver) MakeDir(ctx context.Context, parentDir model.Object, dirName string) error { + //TODO implement me + panic("implement me") +} + +func (d *Driver) Move(ctx context.Context, srcObject, dstDir model.Object) error { + //TODO implement me + panic("implement me") +} + +func (d *Driver) Rename(ctx context.Context, srcObject model.Object, newName string) error { + //TODO implement me + panic("implement me") +} + +func (d *Driver) Copy(ctx context.Context, srcObject, dstDir model.Object) error { + //TODO implement me + panic("implement me") +} + +func (d *Driver) Remove(ctx context.Context, object model.Object) error { + //TODO implement me + panic("implement me") +} + +func (d *Driver) Put(ctx context.Context, parentDir model.Object, stream model.FileStreamer) error { //TODO implement me panic("implement me") } @@ -59,34 +89,4 @@ func (d Driver) Other(ctx context.Context, data interface{}) (interface{}, error panic("implement me") } -func (d *Driver) MakeDir(ctx context.Context, path string) error { - //TODO implement me - panic("implement me") -} - -func (d *Driver) Move(ctx context.Context, src, dst string) error { - //TODO implement me - panic("implement me") -} - -func (d *Driver) Rename(ctx context.Context, src, dst string) error { - //TODO implement me - panic("implement me") -} - -func (d *Driver) Copy(ctx context.Context, src, dst string) error { - //TODO implement me - panic("implement me") -} - -func (d *Driver) Remove(ctx context.Context, path string) error { - //TODO implement me - panic("implement me") -} - -func (d *Driver) Put(ctx context.Context, parentPath string, stream model.FileStreamer) error { - //TODO implement me - panic("implement me") -} - var _ driver.Driver = (*Driver)(nil) diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 9b556ebb..73807775 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -29,16 +29,22 @@ type Other interface { } type Reader interface { - List(ctx context.Context, path string) ([]model.FileInfo, error) - Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, error) + List(ctx context.Context, dir model.Object) ([]model.Object, error) + Link(ctx context.Context, file model.Object, args model.LinkArgs) (*model.Link, error) //Get(ctx context.Context, path string) (FileInfo, error) // maybe not need } type Writer interface { - MakeDir(ctx context.Context, path string) error - Move(ctx context.Context, srcPath, dstPath string) error - Rename(ctx context.Context, srcPath, dstName string) error - Copy(ctx context.Context, srcPath, dstPath string) error - Remove(ctx context.Context, path string) error - Put(ctx context.Context, parentPath string, stream model.FileStreamer) error + // MakeDir make a folder named `dirName` in `parentDir` + MakeDir(ctx context.Context, parentDir model.Object, dirName string) error + // Move move `srcObject` to `dstDir` + Move(ctx context.Context, srcObject, dstDir model.Object) error + // Rename rename `srcObject` to `newName` + Rename(ctx context.Context, srcObject model.Object, newName string) error + // Copy copy `srcObject` to `dstDir` + Copy(ctx context.Context, srcObject, dstDir model.Object) error + // Remove remove `object` + Remove(ctx context.Context, object model.Object) error + // Put put `stream` to `parentDir` + Put(ctx context.Context, parentDir model.Object, stream model.FileStreamer) error } diff --git a/internal/fs/fs.go b/internal/fs/fs.go new file mode 100644 index 00000000..0ae45e87 --- /dev/null +++ b/internal/fs/fs.go @@ -0,0 +1,5 @@ +package fs + +// the param named path of functions in this package is a virtual path +// So, the purpose of this package is to convert virtual path to actual path +// then pass the actual path to the operations package diff --git a/internal/fs/read.go b/internal/fs/read.go index d71bd612..ce9d9393 100644 --- a/internal/fs/read.go +++ b/internal/fs/read.go @@ -15,7 +15,7 @@ import ( // List files // TODO: hide // TODO: sort -func List(ctx context.Context, path string) ([]model.FileInfo, error) { +func List(ctx context.Context, path string) ([]model.Object, error) { account, actualPath, err := operations.GetAccountAndActualPath(path) virtualFiles := operations.GetAccountVirtualFilesByPath(path) if err != nil { @@ -40,7 +40,7 @@ func List(ctx context.Context, path string) ([]model.FileInfo, error) { return files, nil } -func Get(ctx context.Context, path string) (model.FileInfo, error) { +func Get(ctx context.Context, path string) (model.Object, error) { path = utils.StandardizationPath(path) // maybe a virtual file if path != "/" { diff --git a/internal/fs/util.go b/internal/fs/util.go index 0b7dad88..bf7c571e 100644 --- a/internal/fs/util.go +++ b/internal/fs/util.go @@ -12,7 +12,7 @@ import ( "github.com/pkg/errors" ) -func containsByName(files []model.FileInfo, file model.FileInfo) bool { +func containsByName(files []model.Object, file model.Object) bool { for _, f := range files { if f.GetName() == file.GetName() { return true @@ -23,7 +23,7 @@ func containsByName(files []model.FileInfo, file model.FileInfo) bool { var httpClient = &http.Client{} -func getFileStreamFromLink(file model.FileInfo, link *model.Link) (model.FileStreamer, error) { +func getFileStreamFromLink(file model.Object, link *model.Link) (model.FileStreamer, error) { var rc io.ReadCloser mimetype := mime.TypeByExtension(stdpath.Ext(file.GetName())) if link.Data != nil { @@ -57,7 +57,7 @@ func getFileStreamFromLink(file model.FileInfo, link *model.Link) (model.FileStr mimetype = "application/octet-stream" } stream := model.FileStream{ - FileInfo: file, + Object: file, ReadCloser: rc, Mimetype: mimetype, } diff --git a/internal/model/file.go b/internal/model/file.go index 74313212..e8753c07 100644 --- a/internal/model/file.go +++ b/internal/model/file.go @@ -3,6 +3,7 @@ package model import "time" type File struct { + ID string Name string Size uint64 Modified time.Time @@ -25,7 +26,6 @@ func (f File) IsDir() bool { return f.IsFolder } -type FileWithId struct { - Id string - File +func (f File) GetID() string { + return f.ID } diff --git a/internal/model/ifile.go b/internal/model/object.go similarity index 85% rename from internal/model/ifile.go rename to internal/model/object.go index 4a8363af..951aed18 100644 --- a/internal/model/ifile.go +++ b/internal/model/object.go @@ -5,16 +5,17 @@ import ( "time" ) -type FileInfo interface { +type Object interface { GetSize() uint64 GetName() string ModTime() time.Time IsDir() bool + GetID() string } type FileStreamer interface { io.ReadCloser - FileInfo + Object GetMimetype() string } diff --git a/internal/model/stream.go b/internal/model/stream.go index dcbcdb00..90fbe991 100644 --- a/internal/model/stream.go +++ b/internal/model/stream.go @@ -5,7 +5,7 @@ import ( ) type FileStream struct { - FileInfo + Object io.ReadCloser Mimetype string } diff --git a/internal/operations/account.go b/internal/operations/account.go index 3a4352f7..2b949a5d 100644 --- a/internal/operations/account.go +++ b/internal/operations/account.go @@ -136,8 +136,8 @@ func getAccountsByPath(path string) []driver.Driver { // GetAccountVirtualFilesByPath Obtain the virtual file generated by the account according to the path // for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av // GetAccountVirtualFilesByPath(/a) => b,c,d -func GetAccountVirtualFilesByPath(prefix string) []model.FileInfo { - files := make([]model.FileInfo, 0) +func GetAccountVirtualFilesByPath(prefix string) []model.Object { + files := make([]model.Object, 0) accounts := accountsMap.Values() sort.Slice(accounts, func(i, j int) bool { if accounts[i].GetAccount().Index == accounts[j].GetAccount().Index { diff --git a/internal/operations/fs.go b/internal/operations/fs.go index dc706318..6163b343 100644 --- a/internal/operations/fs.go +++ b/internal/operations/fs.go @@ -6,6 +6,7 @@ import ( "time" "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/singleflight" @@ -15,50 +16,57 @@ import ( // In order to facilitate adding some other things before and after file operations -var filesCache = cache.NewMemCache(cache.WithShards[[]model.FileInfo](64)) -var filesG singleflight.Group[[]model.FileInfo] +var filesCache = cache.NewMemCache(cache.WithShards[[]model.Object](64)) +var filesG singleflight.Group[[]model.Object] // List files in storage, not contains virtual file -func List(ctx context.Context, account driver.Driver, path string) ([]model.FileInfo, error) { +func List(ctx context.Context, account driver.Driver, path string) ([]model.Object, error) { + dir, err := Get(ctx, account, path) + if err != nil { + return nil, errors.WithMessage(err, "failed get dir") + } if account.Config().NoCache { - return account.List(ctx, path) + return account.List(ctx, dir) } key := stdpath.Join(account.GetAccount().VirtualPath, path) if files, ok := filesCache.Get(key); ok { return files, nil } - files, err, _ := filesG.Do(key, func() ([]model.FileInfo, error) { - files, err := account.List(ctx, path) + files, err, _ := filesG.Do(key, func() ([]model.Object, error) { + files, err := account.List(ctx, dir) if err != nil { return nil, errors.WithMessage(err, "failed to list files") } - // TODO: get duration from global config or account's config - filesCache.Set(key, files, cache.WithEx[[]model.FileInfo](time.Minute*30)) + // TODO: maybe can get duration from account's config + filesCache.Set(key, files, cache.WithEx[[]model.Object](time.Minute*time.Duration(conf.Conf.CaCheExpiration))) return files, nil }) return files, err } -func Get(ctx context.Context, account driver.Driver, path string) (model.FileInfo, error) { - if r, ok := account.GetAddition().(driver.RootFolderId); ok && utils.PathEqual(path, "/") { - return model.FileWithId{ - Id: r.GetRootFolderId(), - File: model.File{ - Name: "root", - Size: 0, - Modified: account.GetAccount().Modified, - IsFolder: true, - }, - }, nil - } - if r, ok := account.GetAddition().(driver.IRootFolderPath); ok && utils.PathEqual(path, r.GetRootFolderPath()) { +// Get get object from list of files +// TODO: maybe should set object ID with path here +func Get(ctx context.Context, account driver.Driver, path string) (model.Object, error) { + // is root folder + if r, ok := account.GetAddition().(driver.IRootFolderId); ok && utils.PathEqual(path, "/") { return model.File{ + ID: r.GetRootFolderId(), Name: "root", Size: 0, Modified: account.GetAccount().Modified, IsFolder: true, }, nil } + if r, ok := account.GetAddition().(driver.IRootFolderPath); ok && utils.PathEqual(path, r.GetRootFolderPath()) { + return model.File{ + ID: r.GetRootFolderPath(), + Name: "root", + Size: 0, + Modified: account.GetAccount().Modified, + IsFolder: true, + }, nil + } + // not root folder dir, name := stdpath.Split(path) files, err := List(ctx, account, dir) if err != nil { @@ -82,7 +90,11 @@ func Link(ctx context.Context, account driver.Driver, path string, args model.Li return link, nil } fn := func() (*model.Link, error) { - link, err := account.Link(ctx, path, args) + file, err := Get(ctx, account, path) + if err != nil { + return nil, errors.WithMessage(err, "failed to get file") + } + link, err := account.Link(ctx, file, args) if err != nil { return nil, errors.WithMessage(err, "failed get link") } @@ -98,54 +110,81 @@ func Link(ctx context.Context, account driver.Driver, path string, args model.Li func MakeDir(ctx context.Context, account driver.Driver, path string) error { // check if dir exists f, err := Get(ctx, account, path) - if f != nil && f.IsDir() { - return nil - } - if err != nil && !driver.IsErrObjectNotFound(err) { - return errors.WithMessage(err, "failed to check if dir exists") - } - parentPath := stdpath.Dir(path) - err = MakeDir(ctx, account, parentPath) if err != nil { - return errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) - } - return account.MakeDir(ctx, path) -} - -func Move(ctx context.Context, account driver.Driver, srcPath, dstPath string) error { - return account.Move(ctx, srcPath, dstPath) -} - -func Rename(ctx context.Context, account driver.Driver, srcPath, dstName string) error { - return account.Rename(ctx, srcPath, dstName) -} - -// Copy Just copy file[s] in an account -func Copy(ctx context.Context, account driver.Driver, srcPath, dstPath string) error { - return account.Copy(ctx, srcPath, dstPath) -} - -func Remove(ctx context.Context, account driver.Driver, path string) error { - return account.Remove(ctx, path) -} - -func Put(ctx context.Context, account driver.Driver, parentPath string, file model.FileStreamer) error { - f, err := Get(ctx, account, parentPath) - if err != nil { - // if parent dir not exists, create it if driver.IsErrObjectNotFound(err) { + parentPath, dirName := stdpath.Split(path) err = MakeDir(ctx, account, parentPath) if err != nil { return errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) } + parentDir, err := Get(ctx, account, parentPath) + // this should not happen + if err != nil { + return errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + } + return account.MakeDir(ctx, parentDir, dirName) } else { - return errors.WithMessage(err, "failed to get parent dir") + return errors.WithMessage(err, "failed to check if dir exists") } } else { - // object exists, check if it is a dir - if !f.IsDir() { - return errors.Errorf("object [%s] is not a dir", parentPath) + // dir exists + if f.IsDir() { + return nil + } else { + // dir to make is a file + return errors.New("file exists") } } - return account.Put(ctx, parentPath, file) +} + +func Move(ctx context.Context, account driver.Driver, srcPath, dstPath string) error { + srcObject, err := Get(ctx, account, srcPath) + if err != nil { + return errors.WithMessage(err, "failed to get src object") + } + dstDir, err := Get(ctx, account, stdpath.Dir(dstPath)) + return account.Move(ctx, srcObject, dstDir) +} + +func Rename(ctx context.Context, account driver.Driver, srcPath, dstName string) error { + srcObject, err := Get(ctx, account, srcPath) + if err != nil { + return errors.WithMessage(err, "failed to get src object") + } + return account.Rename(ctx, srcObject, dstName) +} + +// Copy Just copy file[s] in an account +func Copy(ctx context.Context, account driver.Driver, srcPath, dstPath string) error { + srcObject, err := Get(ctx, account, srcPath) + if err != nil { + return errors.WithMessage(err, "failed to get src object") + } + dstDir, err := Get(ctx, account, stdpath.Dir(dstPath)) + return account.Copy(ctx, srcObject, dstDir) +} + +func Remove(ctx context.Context, account driver.Driver, path string) error { + object, err := Get(ctx, account, path) + if err != nil { + // if object not found, it's ok + if driver.IsErrObjectNotFound(err) { + return nil + } + return errors.WithMessage(err, "failed to get object") + } + return account.Remove(ctx, object) +} + +func Put(ctx context.Context, account driver.Driver, parentPath string, file model.FileStreamer) error { + err := MakeDir(ctx, account, parentPath) + if err != nil { + return errors.WithMessagef(err, "failed to make dir [%s]", parentPath) + } + parentDir, err := Get(ctx, account, parentPath) + // this should not happen + if err != nil { + return errors.WithMessagef(err, "failed to get dir [%s]", parentPath) + } + return account.Put(ctx, parentDir, file) } diff --git a/internal/operations/path.go b/internal/operations/path.go index c5e94121..984a48d5 100644 --- a/internal/operations/path.go +++ b/internal/operations/path.go @@ -1,22 +1,24 @@ package operations import ( + stdpath "path" + "strings" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "path" - "strings" ) func ActualPath(account driver.Additional, rawPath string) string { if i, ok := account.(driver.IRootFolderPath); ok { - rawPath = path.Join(i.GetRootFolderPath(), rawPath) + rawPath = stdpath.Join(i.GetRootFolderPath(), rawPath) } return utils.StandardizationPath(rawPath) } -// GetAccountAndActualPath Get the corresponding account, and remove the virtual path prefix in path +// GetAccountAndActualPath Get the corresponding account +// for path: remove the virtual path prefix and join the actual root folder if exists func GetAccountAndActualPath(rawPath string) (driver.Driver, string, error) { rawPath = utils.StandardizationPath(rawPath) account := GetBalancedAccount(rawPath)