chore: rename account to storage
This commit is contained in:
Noah Hsu
@ -1,245 +0,0 @@
|
||||
package operations
|
||||
|
||||
import (
|
||||
"context"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/alist-org/alist/v3/internal/db"
|
||||
"github.com/alist-org/alist/v3/internal/driver"
|
||||
"github.com/alist-org/alist/v3/internal/model"
|
||||
"github.com/alist-org/alist/v3/pkg/generic_sync"
|
||||
"github.com/alist-org/alist/v3/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Although the driver type is stored,
|
||||
// there is an account in each driver,
|
||||
// so it should actually be an account, just wrapped by the driver
|
||||
var accountsMap generic_sync.MapOf[string, driver.Driver]
|
||||
|
||||
func GetAccountByVirtualPath(virtualPath string) (driver.Driver, error) {
|
||||
accountDriver, ok := accountsMap.Load(virtualPath)
|
||||
if !ok {
|
||||
return nil, errors.Errorf("no virtual path for an account is: %s", virtualPath)
|
||||
}
|
||||
return accountDriver, nil
|
||||
}
|
||||
|
||||
// CreateAccount Save the account to database so account can get an id
|
||||
// then instantiate corresponding driver and save it in memory
|
||||
func CreateAccount(ctx context.Context, account model.Account) error {
|
||||
account.Modified = time.Now()
|
||||
account.VirtualPath = utils.StandardizePath(account.VirtualPath)
|
||||
var err error
|
||||
// check driver first
|
||||
driverName := account.Driver
|
||||
driverNew, err := GetDriverNew(driverName)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get driver new")
|
||||
}
|
||||
accountDriver := driverNew()
|
||||
// insert account to database
|
||||
err = db.CreateAccount(&account)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed create account in database")
|
||||
}
|
||||
// already has an id
|
||||
err = accountDriver.Init(ctx, account)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed init account but account is already created")
|
||||
}
|
||||
log.Debugf("account %+v is created", accountDriver)
|
||||
accountsMap.Store(account.VirtualPath, accountDriver)
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateAccount update account
|
||||
// get old account first
|
||||
// drop the account then reinitialize
|
||||
func UpdateAccount(ctx context.Context, account model.Account) error {
|
||||
oldAccount, err := db.GetAccountById(account.ID)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get old account")
|
||||
}
|
||||
if oldAccount.Driver != account.Driver {
|
||||
return errors.Errorf("driver cannot be changed")
|
||||
}
|
||||
account.Modified = time.Now()
|
||||
account.VirtualPath = utils.StandardizePath(account.VirtualPath)
|
||||
err = db.UpdateAccount(&account)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed update account in database")
|
||||
}
|
||||
accountDriver, err := GetAccountByVirtualPath(oldAccount.VirtualPath)
|
||||
if oldAccount.VirtualPath != account.VirtualPath {
|
||||
// virtual path renamed, need to drop the account
|
||||
accountsMap.Delete(oldAccount.VirtualPath)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get account driver")
|
||||
}
|
||||
err = accountDriver.Drop(ctx)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed drop account")
|
||||
}
|
||||
err = accountDriver.Init(ctx, account)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed init account")
|
||||
}
|
||||
accountsMap.Store(account.VirtualPath, accountDriver)
|
||||
return nil
|
||||
}
|
||||
|
||||
func DeleteAccountById(ctx context.Context, id uint) error {
|
||||
account, err := db.GetAccountById(id)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get account")
|
||||
}
|
||||
accountDriver, err := GetAccountByVirtualPath(account.VirtualPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get account driver")
|
||||
}
|
||||
// drop the account in the driver
|
||||
if err := accountDriver.Drop(ctx); err != nil {
|
||||
return errors.WithMessage(err, "failed drop account")
|
||||
}
|
||||
// delete the account in the database
|
||||
if err := db.DeleteAccountById(id); err != nil {
|
||||
return errors.WithMessage(err, "failed delete account in database")
|
||||
}
|
||||
// delete the account in the memory
|
||||
accountsMap.Delete(account.VirtualPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustSaveDriverAccount call from specific driver
|
||||
func MustSaveDriverAccount(driver driver.Driver) {
|
||||
err := saveDriverAccount(driver)
|
||||
if err != nil {
|
||||
log.Errorf("failed save driver account: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func saveDriverAccount(driver driver.Driver) error {
|
||||
account := driver.GetAccount()
|
||||
addition := driver.GetAddition()
|
||||
bytes, err := utils.Json.Marshal(addition)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error while marshal addition")
|
||||
}
|
||||
account.Addition = string(bytes)
|
||||
err = db.UpdateAccount(&account)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed update account in database")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAccountsByPath get account by longest match path, contains balance account.
|
||||
// for example, there is /a/b,/a/c,/a/d/e,/a/d/e.balance
|
||||
// GetAccountsByPath(/a/d/e/f) => /a/d/e,/a/d/e.balance
|
||||
func getAccountsByPath(path string) []driver.Driver {
|
||||
accounts := make([]driver.Driver, 0)
|
||||
curSlashCount := 0
|
||||
accountsMap.Range(func(key string, value driver.Driver) bool {
|
||||
virtualPath := utils.GetActualVirtualPath(value.GetAccount().VirtualPath)
|
||||
if virtualPath == "/" {
|
||||
virtualPath = ""
|
||||
}
|
||||
// not this
|
||||
if path != virtualPath && !strings.HasPrefix(path, virtualPath+"/") {
|
||||
return true
|
||||
}
|
||||
slashCount := strings.Count(virtualPath, "/")
|
||||
// not the longest match
|
||||
if slashCount < curSlashCount {
|
||||
return true
|
||||
}
|
||||
if slashCount > curSlashCount {
|
||||
accounts = accounts[:0]
|
||||
curSlashCount = slashCount
|
||||
}
|
||||
accounts = append(accounts, value)
|
||||
return true
|
||||
})
|
||||
// make sure the order is the same for same input
|
||||
sort.Slice(accounts, func(i, j int) bool {
|
||||
return accounts[i].GetAccount().VirtualPath < accounts[j].GetAccount().VirtualPath
|
||||
})
|
||||
return accounts
|
||||
}
|
||||
|
||||
// GetAccountVirtualFilesByPath Obtain the virtual file generated by the account according to the path
|
||||
// for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av
|
||||
// GetAccountVirtualFilesByPath(/a) => b,c,d
|
||||
func GetAccountVirtualFilesByPath(prefix string) []model.Obj {
|
||||
files := make([]model.Obj, 0)
|
||||
accounts := accountsMap.Values()
|
||||
sort.Slice(accounts, func(i, j int) bool {
|
||||
if accounts[i].GetAccount().Index == accounts[j].GetAccount().Index {
|
||||
return accounts[i].GetAccount().VirtualPath < accounts[j].GetAccount().VirtualPath
|
||||
}
|
||||
return accounts[i].GetAccount().Index < accounts[j].GetAccount().Index
|
||||
})
|
||||
prefix = utils.StandardizePath(prefix)
|
||||
if prefix == "/" {
|
||||
prefix = ""
|
||||
}
|
||||
set := make(map[string]interface{})
|
||||
for _, v := range accounts {
|
||||
// TODO should save a balanced account
|
||||
// balance account
|
||||
if utils.IsBalance(v.GetAccount().VirtualPath) {
|
||||
continue
|
||||
}
|
||||
virtualPath := v.GetAccount().VirtualPath
|
||||
if len(virtualPath) <= len(prefix) {
|
||||
continue
|
||||
}
|
||||
// not prefixed with `prefix`
|
||||
if !strings.HasPrefix(virtualPath, prefix+"/") {
|
||||
continue
|
||||
}
|
||||
name := strings.Split(strings.TrimPrefix(virtualPath, prefix), "/")[1]
|
||||
if _, ok := set[name]; ok {
|
||||
continue
|
||||
}
|
||||
files = append(files, model.Object{
|
||||
Name: name,
|
||||
Size: 0,
|
||||
Modified: v.GetAccount().Modified,
|
||||
})
|
||||
set[name] = nil
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
var balanceMap generic_sync.MapOf[string, int]
|
||||
|
||||
// GetBalancedAccount get account by path
|
||||
func GetBalancedAccount(path string) driver.Driver {
|
||||
path = utils.StandardizePath(path)
|
||||
accounts := getAccountsByPath(path)
|
||||
accountNum := len(accounts)
|
||||
switch accountNum {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
return accounts[0]
|
||||
default:
|
||||
virtualPath := utils.GetActualVirtualPath(accounts[0].GetAccount().VirtualPath)
|
||||
cur, ok := balanceMap.Load(virtualPath)
|
||||
i := 0
|
||||
if ok {
|
||||
i = cur
|
||||
i = (i + 1) % accountNum
|
||||
balanceMap.Store(virtualPath, i)
|
||||
} else {
|
||||
balanceMap.Store(virtualPath, i)
|
||||
}
|
||||
return accounts[i]
|
||||
}
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
package operations_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/alist-org/alist/v3/internal/db"
|
||||
"testing"
|
||||
|
||||
"github.com/alist-org/alist/v3/internal/model"
|
||||
"github.com/alist-org/alist/v3/internal/operations"
|
||||
"github.com/alist-org/alist/v3/pkg/utils"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func init() {
|
||||
dB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
|
||||
if err != nil {
|
||||
panic("failed to connect database")
|
||||
}
|
||||
db.Init(dB)
|
||||
}
|
||||
|
||||
func TestCreateAccount(t *testing.T) {
|
||||
var accounts = []struct {
|
||||
account model.Account
|
||||
iserr bool
|
||||
}{
|
||||
{account: model.Account{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, iserr: false},
|
||||
{account: model.Account{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, iserr: true},
|
||||
{account: model.Account{Driver: "None", VirtualPath: "/none", Addition: `{"root_folder":"."}`}, iserr: true},
|
||||
}
|
||||
for _, account := range accounts {
|
||||
err := operations.CreateAccount(context.Background(), account.account)
|
||||
if err != nil {
|
||||
if !account.iserr {
|
||||
t.Errorf("failed to create account: %+v", err)
|
||||
} else {
|
||||
t.Logf("expect failed to create account: %+v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAccountVirtualFilesByPath(t *testing.T) {
|
||||
setupAccounts(t)
|
||||
virtualFiles := operations.GetAccountVirtualFilesByPath("/a")
|
||||
var names []string
|
||||
for _, virtualFile := range virtualFiles {
|
||||
names = append(names, virtualFile.GetName())
|
||||
}
|
||||
var expectedNames = []string{"b", "c", "d"}
|
||||
if utils.SliceEqual(names, expectedNames) {
|
||||
t.Logf("passed")
|
||||
} else {
|
||||
t.Errorf("expected: %+v, got: %+v", expectedNames, names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBalancedAccount(t *testing.T) {
|
||||
setupAccounts(t)
|
||||
account := operations.GetBalancedAccount("/a/d/e")
|
||||
if account.GetAccount().VirtualPath != "/a/d/e" {
|
||||
t.Errorf("expected: /a/d/e, got: %+v", account.GetAccount().VirtualPath)
|
||||
}
|
||||
account = operations.GetBalancedAccount("/a/d/e")
|
||||
if account.GetAccount().VirtualPath != "/a/d/e.balance" {
|
||||
t.Errorf("expected: /a/d/e.balance, got: %+v", account.GetAccount().VirtualPath)
|
||||
}
|
||||
}
|
||||
|
||||
func setupAccounts(t *testing.T) {
|
||||
var accounts = []model.Account{
|
||||
{Driver: "Local", VirtualPath: "/a/b", Index: 0, Addition: `{"root_folder":"."}`},
|
||||
{Driver: "Local", VirtualPath: "/a/c", Index: 1, Addition: `{"root_folder":"."}`},
|
||||
{Driver: "Local", VirtualPath: "/a/d", Index: 2, Addition: `{"root_folder":"."}`},
|
||||
{Driver: "Local", VirtualPath: "/a/d/e", Index: 3, Addition: `{"root_folder":"."}`},
|
||||
{Driver: "Local", VirtualPath: "/a/d/e.balance", Index: 4, Addition: `{"root_folder":"."}`},
|
||||
}
|
||||
for _, account := range accounts {
|
||||
err := operations.CreateAccount(context.Background(), account)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create account: %+v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -23,37 +23,37 @@ import (
|
||||
var filesCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64))
|
||||
var filesG singleflight.Group[[]model.Obj]
|
||||
|
||||
func ClearCache(account driver.Driver, path string) {
|
||||
key := stdpath.Join(account.GetAccount().VirtualPath, path)
|
||||
func ClearCache(storage driver.Driver, path string) {
|
||||
key := stdpath.Join(storage.GetStorage().VirtualPath, path)
|
||||
filesCache.Del(key)
|
||||
}
|
||||
|
||||
// List files in storage, not contains virtual file
|
||||
func List(ctx context.Context, account driver.Driver, path string, refresh ...bool) ([]model.Obj, error) {
|
||||
func List(ctx context.Context, storage driver.Driver, path string, refresh ...bool) ([]model.Obj, error) {
|
||||
path = utils.StandardizePath(path)
|
||||
log.Debugf("operations.List %s", path)
|
||||
dir, err := Get(ctx, account, path)
|
||||
dir, err := Get(ctx, storage, path)
|
||||
if err != nil {
|
||||
return nil, errors.WithMessage(err, "failed get dir")
|
||||
}
|
||||
if !dir.IsDir() {
|
||||
return nil, errors.WithStack(errs.NotFolder)
|
||||
}
|
||||
if account.Config().NoCache {
|
||||
return account.List(ctx, dir)
|
||||
if storage.Config().NoCache {
|
||||
return storage.List(ctx, dir)
|
||||
}
|
||||
key := stdpath.Join(account.GetAccount().VirtualPath, path)
|
||||
key := stdpath.Join(storage.GetStorage().VirtualPath, path)
|
||||
if len(refresh) == 0 || !refresh[0] {
|
||||
if files, ok := filesCache.Get(key); ok {
|
||||
return files, nil
|
||||
}
|
||||
}
|
||||
files, err, _ := filesG.Do(key, func() ([]model.Obj, error) {
|
||||
files, err := account.List(ctx, dir)
|
||||
files, err := storage.List(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, errors.WithMessage(err, "failed to list files")
|
||||
}
|
||||
// TODO: maybe can get duration from account's config
|
||||
// TODO: maybe can get duration from storage's config
|
||||
filesCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(conf.Conf.CaCheExpiration)))
|
||||
return files, nil
|
||||
})
|
||||
@ -74,34 +74,34 @@ func isRoot(path, rootFolderPath string) bool {
|
||||
}
|
||||
|
||||
// Get object from list of files
|
||||
func Get(ctx context.Context, account driver.Driver, path string) (model.Obj, error) {
|
||||
func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {
|
||||
path = utils.StandardizePath(path)
|
||||
log.Debugf("operations.Get %s", path)
|
||||
if g, ok := account.(driver.Getter); ok {
|
||||
if g, ok := storage.(driver.Getter); ok {
|
||||
return g.Get(ctx, path)
|
||||
}
|
||||
// is root folder
|
||||
if r, ok := account.GetAddition().(driver.IRootFolderId); ok && utils.PathEqual(path, "/") {
|
||||
if r, ok := storage.GetAddition().(driver.IRootFolderId); ok && utils.PathEqual(path, "/") {
|
||||
return model.Object{
|
||||
ID: r.GetRootFolderId(),
|
||||
Name: "root",
|
||||
Size: 0,
|
||||
Modified: account.GetAccount().Modified,
|
||||
Modified: storage.GetStorage().Modified,
|
||||
IsFolder: true,
|
||||
}, nil
|
||||
}
|
||||
if r, ok := account.GetAddition().(driver.IRootFolderPath); ok && isRoot(path, r.GetRootFolderPath()) {
|
||||
if r, ok := storage.GetAddition().(driver.IRootFolderPath); ok && isRoot(path, r.GetRootFolderPath()) {
|
||||
return model.Object{
|
||||
ID: r.GetRootFolderPath(),
|
||||
Name: "root",
|
||||
Size: 0,
|
||||
Modified: account.GetAccount().Modified,
|
||||
Modified: storage.GetStorage().Modified,
|
||||
IsFolder: true,
|
||||
}, nil
|
||||
}
|
||||
// not root folder
|
||||
dir, name := stdpath.Split(path)
|
||||
files, err := List(ctx, account, dir)
|
||||
files, err := List(ctx, storage, dir)
|
||||
if err != nil {
|
||||
return nil, errors.WithMessage(err, "failed get parent list")
|
||||
}
|
||||
@ -124,20 +124,20 @@ var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16))
|
||||
var linkG singleflight.Group[*model.Link]
|
||||
|
||||
// Link get link, if is an url. should have an expiry time
|
||||
func Link(ctx context.Context, account driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
|
||||
file, err := Get(ctx, account, path)
|
||||
func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {
|
||||
file, err := Get(ctx, storage, path)
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithMessage(err, "failed to get file")
|
||||
}
|
||||
if file.IsDir() {
|
||||
return nil, nil, errors.WithStack(errs.NotFile)
|
||||
}
|
||||
key := stdpath.Join(account.GetAccount().VirtualPath, path)
|
||||
key := stdpath.Join(storage.GetStorage().VirtualPath, path)
|
||||
if link, ok := linkCache.Get(key); ok {
|
||||
return link, file, nil
|
||||
}
|
||||
fn := func() (*model.Link, error) {
|
||||
link, err := account.Link(ctx, file, args)
|
||||
link, err := storage.Link(ctx, file, args)
|
||||
if err != nil {
|
||||
return nil, errors.WithMessage(err, "failed get link")
|
||||
}
|
||||
@ -150,22 +150,22 @@ func Link(ctx context.Context, account driver.Driver, path string, args model.Li
|
||||
return link, file, err
|
||||
}
|
||||
|
||||
func MakeDir(ctx context.Context, account driver.Driver, path string) error {
|
||||
func MakeDir(ctx context.Context, storage driver.Driver, path string) error {
|
||||
// check if dir exists
|
||||
f, err := Get(ctx, account, path)
|
||||
f, err := Get(ctx, storage, path)
|
||||
if err != nil {
|
||||
if errs.IsObjectNotFound(err) {
|
||||
parentPath, dirName := stdpath.Split(path)
|
||||
err = MakeDir(ctx, account, parentPath)
|
||||
err = MakeDir(ctx, storage, parentPath)
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath)
|
||||
}
|
||||
parentDir, err := Get(ctx, account, parentPath)
|
||||
parentDir, err := Get(ctx, storage, parentPath)
|
||||
// this should not happen
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath)
|
||||
}
|
||||
return account.MakeDir(ctx, parentDir, dirName)
|
||||
return storage.MakeDir(ctx, parentDir, dirName)
|
||||
} else {
|
||||
return errors.WithMessage(err, "failed to check if dir exists")
|
||||
}
|
||||
@ -180,38 +180,38 @@ func MakeDir(ctx context.Context, account driver.Driver, path string) error {
|
||||
}
|
||||
}
|
||||
|
||||
func Move(ctx context.Context, account driver.Driver, srcPath, dstDirPath string) error {
|
||||
srcObj, err := Get(ctx, account, srcPath)
|
||||
func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error {
|
||||
srcObj, err := Get(ctx, storage, srcPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed to get src object")
|
||||
}
|
||||
dstDir, err := Get(ctx, account, dstDirPath)
|
||||
dstDir, err := Get(ctx, storage, dstDirPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed to get dst dir")
|
||||
}
|
||||
return account.Move(ctx, srcObj, dstDir)
|
||||
return storage.Move(ctx, srcObj, dstDir)
|
||||
}
|
||||
|
||||
func Rename(ctx context.Context, account driver.Driver, srcPath, dstName string) error {
|
||||
srcObj, err := Get(ctx, account, srcPath)
|
||||
func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) error {
|
||||
srcObj, err := Get(ctx, storage, srcPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed to get src object")
|
||||
}
|
||||
return account.Rename(ctx, srcObj, dstName)
|
||||
return storage.Rename(ctx, srcObj, dstName)
|
||||
}
|
||||
|
||||
// Copy Just copy file[s] in an account
|
||||
func Copy(ctx context.Context, account driver.Driver, srcPath, dstDirPath string) error {
|
||||
srcObj, err := Get(ctx, account, srcPath)
|
||||
// Copy Just copy file[s] in a storage
|
||||
func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error {
|
||||
srcObj, err := Get(ctx, storage, srcPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed to get src object")
|
||||
}
|
||||
dstDir, err := Get(ctx, account, dstDirPath)
|
||||
return account.Copy(ctx, srcObj, dstDir)
|
||||
dstDir, err := Get(ctx, storage, dstDirPath)
|
||||
return storage.Copy(ctx, srcObj, dstDir)
|
||||
}
|
||||
|
||||
func Remove(ctx context.Context, account driver.Driver, path string) error {
|
||||
obj, err := Get(ctx, account, path)
|
||||
func Remove(ctx context.Context, storage driver.Driver, path string) error {
|
||||
obj, err := Get(ctx, storage, path)
|
||||
if err != nil {
|
||||
// if object not found, it's ok
|
||||
if errs.IsObjectNotFound(err) {
|
||||
@ -219,10 +219,10 @@ func Remove(ctx context.Context, account driver.Driver, path string) error {
|
||||
}
|
||||
return errors.WithMessage(err, "failed to get object")
|
||||
}
|
||||
return account.Remove(ctx, obj)
|
||||
return storage.Remove(ctx, obj)
|
||||
}
|
||||
|
||||
func Put(ctx context.Context, account driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error {
|
||||
func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error {
|
||||
defer func() {
|
||||
if f, ok := file.GetReadCloser().(*os.File); ok {
|
||||
err := os.RemoveAll(f.Name())
|
||||
@ -236,11 +236,11 @@ func Put(ctx context.Context, account driver.Driver, dstDirPath string, file mod
|
||||
log.Errorf("failed to close file streamer, %v", err)
|
||||
}
|
||||
}()
|
||||
err := MakeDir(ctx, account, dstDirPath)
|
||||
err := MakeDir(ctx, storage, dstDirPath)
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath)
|
||||
}
|
||||
parentDir, err := Get(ctx, account, dstDirPath)
|
||||
parentDir, err := Get(ctx, storage, dstDirPath)
|
||||
// this should not happen
|
||||
if err != nil {
|
||||
return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath)
|
||||
@ -249,11 +249,11 @@ func Put(ctx context.Context, account driver.Driver, dstDirPath string, file mod
|
||||
if up == nil {
|
||||
up = func(p int) {}
|
||||
}
|
||||
err = account.Put(ctx, parentDir, file, up)
|
||||
err = storage.Put(ctx, parentDir, file, up)
|
||||
log.Debugf("put file [%s] done", file.GetName())
|
||||
if err == nil {
|
||||
// clear cache
|
||||
key := stdpath.Join(account.GetAccount().VirtualPath, dstDirPath)
|
||||
key := stdpath.Join(storage.GetStorage().VirtualPath, dstDirPath)
|
||||
filesCache.Del(key)
|
||||
}
|
||||
return err
|
||||
|
||||
@ -13,27 +13,27 @@ import (
|
||||
|
||||
// ActualPath Get the actual path
|
||||
// !!! maybe and \ in the path when use windows local
|
||||
func ActualPath(account driver.Additional, rawPath string) string {
|
||||
if i, ok := account.(driver.IRootFolderPath); ok {
|
||||
func ActualPath(storage driver.Additional, rawPath string) string {
|
||||
if i, ok := storage.(driver.IRootFolderPath); ok {
|
||||
rawPath = stdpath.Join(i.GetRootFolderPath(), rawPath)
|
||||
}
|
||||
return utils.StandardizePath(rawPath)
|
||||
}
|
||||
|
||||
// GetAccountAndActualPath Get the corresponding account
|
||||
// GetStorageAndActualPath Get the corresponding storage and actual path
|
||||
// for path: remove the virtual path prefix and join the actual root folder if exists
|
||||
func GetAccountAndActualPath(rawPath string) (driver.Driver, string, error) {
|
||||
func GetStorageAndActualPath(rawPath string) (driver.Driver, string, error) {
|
||||
rawPath = utils.StandardizePath(rawPath)
|
||||
if strings.Contains(rawPath, "..") {
|
||||
return nil, "", errors.WithStack(errs.RelativePath)
|
||||
}
|
||||
account := GetBalancedAccount(rawPath)
|
||||
if account == nil {
|
||||
return nil, "", errors.Errorf("can't find account with rawPath: %s", rawPath)
|
||||
storage := GetBalancedStorage(rawPath)
|
||||
if storage == nil {
|
||||
return nil, "", errors.Errorf("can't find storage with rawPath: %s", rawPath)
|
||||
}
|
||||
log.Debugln("use account: ", account.GetAccount().VirtualPath)
|
||||
virtualPath := utils.GetActualVirtualPath(account.GetAccount().VirtualPath)
|
||||
log.Debugln("use storage: ", storage.GetStorage().VirtualPath)
|
||||
virtualPath := utils.GetActualVirtualPath(storage.GetStorage().VirtualPath)
|
||||
actualPath := strings.TrimPrefix(rawPath, virtualPath)
|
||||
actualPath = ActualPath(account.GetAddition(), actualPath)
|
||||
return account, actualPath, nil
|
||||
actualPath = ActualPath(storage.GetAddition(), actualPath)
|
||||
return storage, actualPath, nil
|
||||
}
|
||||
|
||||
245
internal/operations/storage.go
Normal file
245
internal/operations/storage.go
Normal file
@ -0,0 +1,245 @@
|
||||
package operations
|
||||
|
||||
import (
|
||||
"context"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/alist-org/alist/v3/internal/db"
|
||||
"github.com/alist-org/alist/v3/internal/driver"
|
||||
"github.com/alist-org/alist/v3/internal/model"
|
||||
"github.com/alist-org/alist/v3/pkg/generic_sync"
|
||||
"github.com/alist-org/alist/v3/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Although the driver type is stored,
|
||||
// there is a storage in each driver,
|
||||
// so it should actually be a storage, just wrapped by the driver
|
||||
var storagesMap generic_sync.MapOf[string, driver.Driver]
|
||||
|
||||
func GetStorageByVirtualPath(virtualPath string) (driver.Driver, error) {
|
||||
storageDriver, ok := storagesMap.Load(virtualPath)
|
||||
if !ok {
|
||||
return nil, errors.Errorf("no virtual path for an storage is: %s", virtualPath)
|
||||
}
|
||||
return storageDriver, nil
|
||||
}
|
||||
|
||||
// CreateStorage Save the storage to database so storage can get an id
|
||||
// then instantiate corresponding driver and save it in memory
|
||||
func CreateStorage(ctx context.Context, storage model.Storage) error {
|
||||
storage.Modified = time.Now()
|
||||
storage.VirtualPath = utils.StandardizePath(storage.VirtualPath)
|
||||
var err error
|
||||
// check driver first
|
||||
driverName := storage.Driver
|
||||
driverNew, err := GetDriverNew(driverName)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get driver new")
|
||||
}
|
||||
storageDriver := driverNew()
|
||||
// insert storage to database
|
||||
err = db.CreateStorage(&storage)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed create storage in database")
|
||||
}
|
||||
// already has an id
|
||||
err = storageDriver.Init(ctx, storage)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed init storage but storage is already created")
|
||||
}
|
||||
log.Debugf("storage %+v is created", storageDriver)
|
||||
storagesMap.Store(storage.VirtualPath, storageDriver)
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateStorage update storage
|
||||
// get old storage first
|
||||
// drop the storage then reinitialize
|
||||
func UpdateStorage(ctx context.Context, storage model.Storage) error {
|
||||
oldStorage, err := db.GetStorageById(storage.ID)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get old storage")
|
||||
}
|
||||
if oldStorage.Driver != storage.Driver {
|
||||
return errors.Errorf("driver cannot be changed")
|
||||
}
|
||||
storage.Modified = time.Now()
|
||||
storage.VirtualPath = utils.StandardizePath(storage.VirtualPath)
|
||||
err = db.UpdateStorage(&storage)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed update storage in database")
|
||||
}
|
||||
storageDriver, err := GetStorageByVirtualPath(oldStorage.VirtualPath)
|
||||
if oldStorage.VirtualPath != storage.VirtualPath {
|
||||
// virtual path renamed, need to drop the storage
|
||||
storagesMap.Delete(oldStorage.VirtualPath)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get storage driver")
|
||||
}
|
||||
err = storageDriver.Drop(ctx)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed drop storage")
|
||||
}
|
||||
err = storageDriver.Init(ctx, storage)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed init storage")
|
||||
}
|
||||
storagesMap.Store(storage.VirtualPath, storageDriver)
|
||||
return nil
|
||||
}
|
||||
|
||||
func DeleteStorageById(ctx context.Context, id uint) error {
|
||||
storage, err := db.GetStorageById(id)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get storage")
|
||||
}
|
||||
storageDriver, err := GetStorageByVirtualPath(storage.VirtualPath)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed get storage driver")
|
||||
}
|
||||
// drop the storage in the driver
|
||||
if err := storageDriver.Drop(ctx); err != nil {
|
||||
return errors.WithMessage(err, "failed drop storage")
|
||||
}
|
||||
// delete the storage in the database
|
||||
if err := db.DeleteStorageById(id); err != nil {
|
||||
return errors.WithMessage(err, "failed delete storage in database")
|
||||
}
|
||||
// delete the storage in the memory
|
||||
storagesMap.Delete(storage.VirtualPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustSaveDriverStorage call from specific driver
|
||||
func MustSaveDriverStorage(driver driver.Driver) {
|
||||
err := saveDriverStorage(driver)
|
||||
if err != nil {
|
||||
log.Errorf("failed save driver storage: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func saveDriverStorage(driver driver.Driver) error {
|
||||
storage := driver.GetStorage()
|
||||
addition := driver.GetAddition()
|
||||
bytes, err := utils.Json.Marshal(addition)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error while marshal addition")
|
||||
}
|
||||
storage.Addition = string(bytes)
|
||||
err = db.UpdateStorage(&storage)
|
||||
if err != nil {
|
||||
return errors.WithMessage(err, "failed update storage in database")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// getStoragesByPath get storage by longest match path, contains balance storage.
|
||||
// for example, there is /a/b,/a/c,/a/d/e,/a/d/e.balance
|
||||
// getStoragesByPath(/a/d/e/f) => /a/d/e,/a/d/e.balance
|
||||
func getStoragesByPath(path string) []driver.Driver {
|
||||
storages := make([]driver.Driver, 0)
|
||||
curSlashCount := 0
|
||||
storagesMap.Range(func(key string, value driver.Driver) bool {
|
||||
virtualPath := utils.GetActualVirtualPath(value.GetStorage().VirtualPath)
|
||||
if virtualPath == "/" {
|
||||
virtualPath = ""
|
||||
}
|
||||
// not this
|
||||
if path != virtualPath && !strings.HasPrefix(path, virtualPath+"/") {
|
||||
return true
|
||||
}
|
||||
slashCount := strings.Count(virtualPath, "/")
|
||||
// not the longest match
|
||||
if slashCount < curSlashCount {
|
||||
return true
|
||||
}
|
||||
if slashCount > curSlashCount {
|
||||
storages = storages[:0]
|
||||
curSlashCount = slashCount
|
||||
}
|
||||
storages = append(storages, value)
|
||||
return true
|
||||
})
|
||||
// make sure the order is the same for same input
|
||||
sort.Slice(storages, func(i, j int) bool {
|
||||
return storages[i].GetStorage().VirtualPath < storages[j].GetStorage().VirtualPath
|
||||
})
|
||||
return storages
|
||||
}
|
||||
|
||||
// GetStorageVirtualFilesByPath Obtain the virtual file generated by the storage according to the path
|
||||
// for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av
|
||||
// GetStorageVirtualFilesByPath(/a) => b,c,d
|
||||
func GetStorageVirtualFilesByPath(prefix string) []model.Obj {
|
||||
files := make([]model.Obj, 0)
|
||||
storages := storagesMap.Values()
|
||||
sort.Slice(storages, func(i, j int) bool {
|
||||
if storages[i].GetStorage().Index == storages[j].GetStorage().Index {
|
||||
return storages[i].GetStorage().VirtualPath < storages[j].GetStorage().VirtualPath
|
||||
}
|
||||
return storages[i].GetStorage().Index < storages[j].GetStorage().Index
|
||||
})
|
||||
prefix = utils.StandardizePath(prefix)
|
||||
if prefix == "/" {
|
||||
prefix = ""
|
||||
}
|
||||
set := make(map[string]interface{})
|
||||
for _, v := range storages {
|
||||
// TODO should save a balanced storage
|
||||
// balance storage
|
||||
if utils.IsBalance(v.GetStorage().VirtualPath) {
|
||||
continue
|
||||
}
|
||||
virtualPath := v.GetStorage().VirtualPath
|
||||
if len(virtualPath) <= len(prefix) {
|
||||
continue
|
||||
}
|
||||
// not prefixed with `prefix`
|
||||
if !strings.HasPrefix(virtualPath, prefix+"/") {
|
||||
continue
|
||||
}
|
||||
name := strings.Split(strings.TrimPrefix(virtualPath, prefix), "/")[1]
|
||||
if _, ok := set[name]; ok {
|
||||
continue
|
||||
}
|
||||
files = append(files, model.Object{
|
||||
Name: name,
|
||||
Size: 0,
|
||||
Modified: v.GetStorage().Modified,
|
||||
})
|
||||
set[name] = nil
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
var balanceMap generic_sync.MapOf[string, int]
|
||||
|
||||
// GetBalancedStorage get storage by path
|
||||
func GetBalancedStorage(path string) driver.Driver {
|
||||
path = utils.StandardizePath(path)
|
||||
storages := getStoragesByPath(path)
|
||||
storageNum := len(storages)
|
||||
switch storageNum {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
return storages[0]
|
||||
default:
|
||||
virtualPath := utils.GetActualVirtualPath(storages[0].GetStorage().VirtualPath)
|
||||
cur, ok := balanceMap.Load(virtualPath)
|
||||
i := 0
|
||||
if ok {
|
||||
i = cur
|
||||
i = (i + 1) % storageNum
|
||||
balanceMap.Store(virtualPath, i)
|
||||
} else {
|
||||
balanceMap.Store(virtualPath, i)
|
||||
}
|
||||
return storages[i]
|
||||
}
|
||||
}
|
||||
85
internal/operations/storage_test.go
Normal file
85
internal/operations/storage_test.go
Normal file
@ -0,0 +1,85 @@
|
||||
package operations_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/alist-org/alist/v3/internal/db"
|
||||
"testing"
|
||||
|
||||
"github.com/alist-org/alist/v3/internal/model"
|
||||
"github.com/alist-org/alist/v3/internal/operations"
|
||||
"github.com/alist-org/alist/v3/pkg/utils"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func init() {
|
||||
dB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
|
||||
if err != nil {
|
||||
panic("failed to connect database")
|
||||
}
|
||||
db.Init(dB)
|
||||
}
|
||||
|
||||
func TestCreateStorage(t *testing.T) {
|
||||
var storages = []struct {
|
||||
storage model.Storage
|
||||
isErr bool
|
||||
}{
|
||||
{storage: model.Storage{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, isErr: false},
|
||||
{storage: model.Storage{Driver: "Local", VirtualPath: "/local", Addition: `{"root_folder":"."}`}, isErr: true},
|
||||
{storage: model.Storage{Driver: "None", VirtualPath: "/none", Addition: `{"root_folder":"."}`}, isErr: true},
|
||||
}
|
||||
for _, storage := range storages {
|
||||
err := operations.CreateStorage(context.Background(), storage.storage)
|
||||
if err != nil {
|
||||
if !storage.isErr {
|
||||
t.Errorf("failed to create storage: %+v", err)
|
||||
} else {
|
||||
t.Logf("expect failed to create storage: %+v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetStorageVirtualFilesByPath(t *testing.T) {
|
||||
setupStorages(t)
|
||||
virtualFiles := operations.GetStorageVirtualFilesByPath("/a")
|
||||
var names []string
|
||||
for _, virtualFile := range virtualFiles {
|
||||
names = append(names, virtualFile.GetName())
|
||||
}
|
||||
var expectedNames = []string{"b", "c", "d"}
|
||||
if utils.SliceEqual(names, expectedNames) {
|
||||
t.Logf("passed")
|
||||
} else {
|
||||
t.Errorf("expected: %+v, got: %+v", expectedNames, names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBalancedStorage(t *testing.T) {
|
||||
setupStorages(t)
|
||||
storage := operations.GetBalancedStorage("/a/d/e")
|
||||
if storage.GetStorage().VirtualPath != "/a/d/e" {
|
||||
t.Errorf("expected: /a/d/e, got: %+v", storage.GetStorage().VirtualPath)
|
||||
}
|
||||
storage = operations.GetBalancedStorage("/a/d/e")
|
||||
if storage.GetStorage().VirtualPath != "/a/d/e.balance" {
|
||||
t.Errorf("expected: /a/d/e.balance, got: %+v", storage.GetStorage().VirtualPath)
|
||||
}
|
||||
}
|
||||
|
||||
func setupStorages(t *testing.T) {
|
||||
var storages = []model.Storage{
|
||||
{Driver: "Local", VirtualPath: "/a/b", Index: 0, Addition: `{"root_folder":"."}`},
|
||||
{Driver: "Local", VirtualPath: "/a/c", Index: 1, Addition: `{"root_folder":"."}`},
|
||||
{Driver: "Local", VirtualPath: "/a/d", Index: 2, Addition: `{"root_folder":"."}`},
|
||||
{Driver: "Local", VirtualPath: "/a/d/e", Index: 3, Addition: `{"root_folder":"."}`},
|
||||
{Driver: "Local", VirtualPath: "/a/d/e.balance", Index: 4, Addition: `{"root_folder":"."}`},
|
||||
}
|
||||
for _, storage := range storages {
|
||||
err := operations.CreateStorage(context.Background(), storage)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create storage: %+v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user