ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: split the db package hook and cache to the op package (#2747)

Browse Source 
* refactor:separate the setting method from the db package to the op package and add the cache

* refactor:separate the meta method from the db package to the op package

* fix:setting not load database data

* refactor:separate the user method from the db package to the op package

* refactor:remove user JoinPath error

* fix:op package user cache

* refactor:fs package list method

* fix:tile virtual paths (close #2743)

* Revert "refactor:remove user JoinPath error"

This reverts commit 4e20daaf9e700da047000d4fd4900abbe05c3848.

* clean path directly may lead to unknown behavior

* fix: The path of the meta passed in must be prefix of reqPath

* chore: rename all virtualPath to mountPath

* fix: `getStoragesByPath` and `GetStorageVirtualFilesByPath`

is_sub_path:

/a/b isn't subpath of /a/bc

* fix: don't save setting if hook error

Co-authored-by: Noah Hsu <i@nn.ci>
This commit is contained in:
foxxorcat
2022-12-18 19:51:20 +08:00
committed by GitHub
parent f38f4f401b
commit 6024e8d832
39 changed files with 741 additions and 543 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
internal/op/hook.go
View File

@ -1,7 +1,17 @@
package op
import "github.com/alist-org/alist/v3/internal/model"
import (
"regexp"
"strings"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
// Obj
type ObjsUpdateHook = func(parent string, objs []model.Obj)
var (
@ -11,3 +21,78 @@ var (
func RegisterObjsUpdateHook(hook ObjsUpdateHook) {
objsUpdateHooks = append(objsUpdateHooks, hook)
}
func HandleObjsUpdateHook(parent string, objs []model.Obj) {
for _, hook := range objsUpdateHooks {
hook(parent, objs)
}
}
// Setting
type SettingItemHook func(item *model.SettingItem) error
var settingItemHooks = map[string]SettingItemHook{
conf.VideoTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
return nil
},
conf.AudioTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ImageTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
return nil
},
conf.TextTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ProxyTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
return nil
},
conf.PrivacyRegs: func(item *model.SettingItem) error {
regStrs := strings.Split(item.Value, "\n")
regs := make([]*regexp.Regexp, 0, len(regStrs))
for _, regStr := range regStrs {
reg, err := regexp.Compile(regStr)
if err != nil {
return errors.WithStack(err)
}
regs = append(regs, reg)
}
conf.PrivacyReg = regs
return nil
},
conf.FilenameCharMapping: func(item *model.SettingItem) error {
err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
if err != nil {
return err
}
log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
return nil
},
}
func RegisterSettingItemHook(key string, hook SettingItemHook) {
settingItemHooks[key] = hook
}
func HandleSettingItemHook(item *model.SettingItem) (hasHook bool, err error) {
if hook, ok := settingItemHooks[item.Key]; ok {
return true, hook(item)
}
return false, nil
}
//func HandleSettingItemsHook(items []model.SettingItem) (err error) {
// for i := 0; i < len(items); i++ {
// _, err = HandleSettingItemHook(&items[i])
// if err != nil {
// return err
// }
// }
// return nil
//}

88
internal/op/meta.go Normal file
View File

@ -0,0 +1,88 @@
package op
import (
stdpath "path"
"time"
"github.com/Xhofe/go-cache"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/singleflight"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
"gorm.io/gorm"
)
var metaCache = cache.NewMemCache(cache.WithShards[*model.Meta](2))
// metaG maybe not needed
var metaG singleflight.Group[*model.Meta]
func GetNearestMeta(path string) (*model.Meta, error) {
return getNearestMeta(utils.FixAndCleanPath(path))
}
func getNearestMeta(path string) (*model.Meta, error) {
meta, err := GetMetaByPath(path)
if err == nil {
return meta, nil
}
if errors.Cause(err) != gorm.ErrRecordNotFound {
return nil, err
}
if path == "/" {
return nil, errs.MetaNotFound
}
return getNearestMeta(stdpath.Dir(path))
}
func GetMetaByPath(path string) (*model.Meta, error) {
return getMetaByPath(utils.FixAndCleanPath(path))
}
func getMetaByPath(path string) (*model.Meta, error) {
meta, ok := metaCache.Get(path)
if ok {
return meta, nil
}
meta, err, _ := metaG.Do(path, func() (*model.Meta, error) {
_meta, err := db.GetMetaByPath(path)
if err != nil {
return nil, err
}
metaCache.Set(path, _meta, cache.WithEx[*model.Meta](time.Hour))
return _meta, nil
})
return meta, err
}
func DeleteMetaById(id uint) error {
old, err := db.GetMetaById(id)
if err != nil {
return err
}
metaCache.Del(old.Path)
return db.DeleteMetaById(id)
}
func UpdateMeta(u *model.Meta) error {
u.Path = utils.FixAndCleanPath(u.Path)
old, err := db.GetMetaById(u.ID)
if err != nil {
return err
}
metaCache.Del(old.Path)
return db.UpdateMeta(u)
}
func CreateMeta(u *model.Meta) error {
u.Path = utils.FixAndCleanPath(u.Path)
return db.CreateMeta(u)
}
func GetMetaById(id uint) (*model.Meta, error) {
return db.GetMetaById(id)
}
func GetMetas(pageIndex, pageSize int) (metas []model.Meta, count int64, err error) {
return db.GetMetas(pageIndex, pageSize)
}

4
internal/op/path.go
View File

@ -19,7 +19,7 @@ func GetStorageAndActualPath(rawPath string) (storage driver.Driver, actualPath
return
}
log.Debugln("use storage: ", storage.GetStorage().MountPath)
virtualPath := utils.GetActualVirtualPath(storage.GetStorage().MountPath)
actualPath = utils.FixAndCleanPath(strings.TrimPrefix(rawPath, virtualPath))
mountPath := utils.GetActualMountPath(storage.GetStorage().MountPath)
actualPath = utils.FixAndCleanPath(strings.TrimPrefix(rawPath, mountPath))
return
}

198
internal/op/setting.go Normal file
View File

@ -0,0 +1,198 @@
package op
import (
"sort"
"strconv"
"strings"
"time"
"github.com/Xhofe/go-cache"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/singleflight"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
)
var settingCache = cache.NewMemCache(cache.WithShards[*model.SettingItem](4))
var settingG singleflight.Group[*model.SettingItem]
var settingCacheF = func(item *model.SettingItem) {
settingCache.Set(item.Key, item, cache.WithEx[*model.SettingItem](time.Hour))
}
var settingGroupCache = cache.NewMemCache(cache.WithShards[[]model.SettingItem](4))
var settingGroupG singleflight.Group[[]model.SettingItem]
var settingGroupCacheF = func(key string, item []model.SettingItem) {
settingGroupCache.Set(key, item, cache.WithEx[[]model.SettingItem](time.Hour))
}
func settingCacheUpdate() {
settingCache.Clear()
settingGroupCache.Clear()
}
func GetPublicSettingsMap() map[string]string {
items, _ := GetPublicSettingItems()
pSettings := make(map[string]string)
for _, item := range items {
pSettings[item.Key] = item.Value
}
return pSettings
}
func GetSettingsMap() map[string]string {
items, _ := GetSettingItems()
settings := make(map[string]string)
for _, item := range items {
settings[item.Key] = item.Value
}
return settings
}
func GetSettingItems() ([]model.SettingItem, error) {
if items, ok := settingGroupCache.Get("ALL_SETTING_ITEMS"); ok {
return items, nil
}
items, err, _ := settingGroupG.Do("ALL_SETTING_ITEMS", func() ([]model.SettingItem, error) {
_items, err := db.GetSettingItems()
if err != nil {
return nil, err
}
settingGroupCacheF("ALL_SETTING_ITEMS", _items)
return _items, nil
})
return items, err
}
func GetPublicSettingItems() ([]model.SettingItem, error) {
if items, ok := settingGroupCache.Get("ALL_PUBLIC_SETTING_ITEMS"); ok {
return items, nil
}
items, err, _ := settingGroupG.Do("ALL_PUBLIC_SETTING_ITEMS", func() ([]model.SettingItem, error) {
_items, err := db.GetPublicSettingItems()
if err != nil {
return nil, err
}
settingGroupCacheF("ALL_PUBLIC_SETTING_ITEMS", _items)
return _items, nil
})
return items, err
}
func GetSettingItemByKey(key string) (*model.SettingItem, error) {
if item, ok := settingCache.Get(key); ok {
return item, nil
}
item, err, _ := settingG.Do(key, func() (*model.SettingItem, error) {
_item, err := db.GetSettingItemByKey(key)
if err != nil {
return nil, err
}
settingCacheF(_item)
return _item, nil
})
return item, err
}
func GetSettingItemInKeys(keys []string) ([]model.SettingItem, error) {
var items []model.SettingItem
for _, key := range keys {
item, err := GetSettingItemByKey(key)
if err != nil {
return nil, err
}
items = append(items, *item)
}
return items, nil
}
func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) {
key := strconv.Itoa(group)
if items, ok := settingGroupCache.Get(key); ok {
return items, nil
}
items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) {
_items, err := db.GetSettingItemsByGroup(group)
if err != nil {
return nil, err
}
settingGroupCacheF(key, _items)
return _items, nil
})
return items, err
}
func GetSettingItemsInGroups(groups []int) ([]model.SettingItem, error) {
sort.Ints(groups)
key := strings.Join(utils.MustSliceConvert(groups, func(i int) string {
return strconv.Itoa(i)
}), ",")
if items, ok := settingGroupCache.Get(key); ok {
return items, nil
}
items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) {
_items, err := db.GetSettingItemsInGroups(groups)
if err != nil {
return nil, err
}
settingGroupCacheF(key, _items)
return _items, nil
})
return items, err
}
func SaveSettingItems(items []model.SettingItem) error {
noHookItems := make([]model.SettingItem, 0)
errs := make([]error, 0)
for i := range items {
if ok, err := HandleSettingItemHook(&items[i]); ok {
if err != nil {
errs = append(errs, err)
} else {
err = db.SaveSettingItem(&items[i])
if err != nil {
errs = append(errs, err)
}
}
} else {
noHookItems = append(noHookItems, items[i])
}
}
if len(noHookItems) > 0 {
err := db.SaveSettingItems(noHookItems)
if err != nil {
errs = append(errs, err)
}
}
if len(errs) < len(items)-len(noHookItems)+1 {
settingCacheUpdate()
}
return utils.MergeErrors(errs...)
}
func SaveSettingItem(item *model.SettingItem) (err error) {
// hook
if _, err := HandleSettingItemHook(item); err != nil {
return err
}
// update
if err = db.SaveSettingItem(item); err != nil {
return err
}
settingCacheUpdate()
return nil
}
func DeleteSettingItemByKey(key string) error {
old, err := GetSettingItemByKey(key)
if err != nil {
return errors.WithMessage(err, "failed to get settingItem")
}
if !old.IsDeprecated() {
return errors.Errorf("setting [%s] is not deprecated", key)
}
settingCacheUpdate()
return db.DeleteSettingItemByKey(key)
}

27
internal/op/storage.go
View File

@ -29,7 +29,7 @@ func HasStorage(mountPath string) bool {
return storagesMap.Has(utils.FixAndCleanPath(mountPath))
}
func GetStorageByVirtualPath(virtualPath string) (driver.Driver, error) {
func GetStorageByMountPath(virtualPath string) (driver.Driver, error) {
virtualPath = utils.FixAndCleanPath(virtualPath)
storageDriver, ok := storagesMap.Load(virtualPath)
if !ok {
@ -130,7 +130,7 @@ func DisableStorage(ctx context.Context, id uint) error {
if storage.Disabled {
return errors.Errorf("this storage have disabled")
}
storageDriver, err := GetStorageByVirtualPath(storage.MountPath)
storageDriver, err := GetStorageByMountPath(storage.MountPath)
if err != nil {
return errors.WithMessage(err, "failed get storage driver")
}
@ -168,7 +168,7 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error {
if storage.Disabled {
return nil
}
storageDriver, err := GetStorageByVirtualPath(oldStorage.MountPath)
storageDriver, err := GetStorageByMountPath(oldStorage.MountPath)
if oldStorage.MountPath != storage.MountPath {
// mount path renamed, need to drop the storage
storagesMap.Delete(oldStorage.MountPath)
@ -192,7 +192,7 @@ func DeleteStorageById(ctx context.Context, id uint) error {
return errors.WithMessage(err, "failed get storage")
}
if !storage.Disabled {
storageDriver, err := GetStorageByVirtualPath(storage.MountPath)
storageDriver, err := GetStorageByMountPath(storage.MountPath)
if err != nil {
return errors.WithMessage(err, "failed get storage driver")
}
@ -221,11 +221,11 @@ func MustSaveDriverStorage(driver driver.Driver) {
func saveDriverStorage(driver driver.Driver) error {
storage := driver.GetStorage()
addition := driver.GetAddition()
strs, err := utils.Json.MarshalToString(addition)
str, err := utils.Json.MarshalToString(addition)
if err != nil {
return errors.Wrap(err, "error while marshal addition")
}
storage.Addition = strs
storage.Addition = str
err = db.UpdateStorage(storage)
if err != nil {
return errors.WithMessage(err, "failed update storage in database")
@ -240,10 +240,10 @@ func getStoragesByPath(path string) []driver.Driver {
storages := make([]driver.Driver, 0)
curSlashCount := 0
storagesMap.Range(func(mountPath string, value driver.Driver) bool {
virtualPath := utils.GetActualVirtualPath(mountPath)
mountPath = utils.GetActualMountPath(mountPath)
// is this path
if strings.HasPrefix(path, virtualPath) {
slashCount := strings.Count(utils.PathAddSeparatorSuffix(virtualPath), "/")
if utils.IsSubPath(path, mountPath) {
slashCount := strings.Count(utils.PathAddSeparatorSuffix(mountPath), "/")
// not the longest match
if slashCount > curSlashCount {
storages = storages[:0]
@ -278,13 +278,12 @@ func GetStorageVirtualFilesByPath(prefix string) []model.Obj {
prefix = utils.FixAndCleanPath(prefix)
set := mapset.NewSet[string]()
for _, v := range storages {
virtualPath := utils.GetActualVirtualPath(v.GetStorage().MountPath)
mountPath := utils.GetActualMountPath(v.GetStorage().MountPath)
// Exclude prefix itself and non prefix
if len(prefix) >= len(virtualPath) || !strings.HasPrefix(virtualPath, prefix) {
if len(prefix) >= len(mountPath) || !utils.IsSubPath(mountPath, prefix) {
continue
}
name := strings.SplitN(strings.TrimPrefix(virtualPath[len(prefix):], "/"), "/", 1)[0]
name := strings.SplitN(strings.TrimPrefix(mountPath[len(prefix):], "/"), "/", 2)[0]
if set.Add(name) {
files = append(files, &model.Object{
Name: name,
@ -310,7 +309,7 @@ func GetBalancedStorage(path string) driver.Driver {
case 1:
return storages[0]
default:
virtualPath := utils.GetActualVirtualPath(storages[0].GetStorage().MountPath)
virtualPath := utils.GetActualMountPath(storages[0].GetStorage().MountPath)
i, _ := balanceMap.LoadOrStore(virtualPath, 0)
i = (i + 1) % storageNum
balanceMap.Store(virtualPath, i)

34
internal/op/storage_test.go
View File

@ -4,10 +4,12 @@ import (
"context"
"testing"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/utils"
mapset "github.com/deckarep/golang-set/v2"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
@ -17,6 +19,7 @@ func init() {
if err != nil {
panic("failed to connect database")
}
conf.Conf = conf.DefaultConfig()
db.Init(dB)
}
@ -25,9 +28,9 @@ func TestCreateStorage(t *testing.T) {
storage model.Storage
isErr bool
}{
{storage: model.Storage{Driver: "Local", MountPath: "/local", Addition: `{"root_folder":"."}`}, isErr: false},
{storage: model.Storage{Driver: "Local", MountPath: "/local", Addition: `{"root_folder":"."}`}, isErr: true},
{storage: model.Storage{Driver: "None", MountPath: "/none", Addition: `{"root_folder":"."}`}, isErr: true},
{storage: model.Storage{Driver: "Local", MountPath: "/local", Addition: `{"root_folder_path":"."}`}, isErr: false},
{storage: model.Storage{Driver: "Local", MountPath: "/local", Addition: `{"root_folder_path":"."}`}, isErr: true},
{storage: model.Storage{Driver: "None", MountPath: "/none", Addition: `{"root_folder_path":"."}`}, isErr: true},
}
for _, storage := range storages {
_, err := op.CreateStorage(context.Background(), storage.storage)
@ -58,23 +61,26 @@ func TestGetStorageVirtualFilesByPath(t *testing.T) {
func TestGetBalancedStorage(t *testing.T) {
setupStorages(t)
storage := op.GetBalancedStorage("/a/d/e")
if storage.GetStorage().MountPath != "/a/d/e" {
t.Errorf("expected: /a/d/e, got: %+v", storage.GetStorage().MountPath)
set := mapset.NewSet[string]()
for i := 0; i < 5; i++ {
storage := op.GetBalancedStorage("/a/d/e1")
set.Add(storage.GetStorage().MountPath)
}
storage = op.GetBalancedStorage("/a/d/e")
if storage.GetStorage().MountPath != "/a/d/e.balance" {
t.Errorf("expected: /a/d/e.balance, got: %+v", storage.GetStorage().MountPath)
expected := mapset.NewSet([]string{"/a/d/e1", "/a/d/e1.balance"}...)
if !expected.Equal(set) {
t.Errorf("expected: %+v, got: %+v", expected, set)
}
}
func setupStorages(t *testing.T) {
var storages = []model.Storage{
{Driver: "Local", MountPath: "/a/b", Order: 0, Addition: `{"root_folder":"."}`},
{Driver: "Local", MountPath: "/a/c", Order: 1, Addition: `{"root_folder":"."}`},
{Driver: "Local", MountPath: "/a/d", Order: 2, Addition: `{"root_folder":"."}`},
{Driver: "Local", MountPath: "/a/d/e", Order: 3, Addition: `{"root_folder":"."}`},
{Driver: "Local", MountPath: "/a/d/e.balance", Order: 4, Addition: `{"root_folder":"."}`},
{Driver: "Local", MountPath: "/a/b", Order: 0, Addition: `{"root_folder_path":"."}`},
{Driver: "Local", MountPath: "/adc", Order: 0, Addition: `{"root_folder_path":"."}`},
{Driver: "Local", MountPath: "/a/c", Order: 1, Addition: `{"root_folder_path":"."}`},
{Driver: "Local", MountPath: "/a/d", Order: 2, Addition: `{"root_folder_path":"."}`},
{Driver: "Local", MountPath: "/a/d/e1", Order: 3, Addition: `{"root_folder_path":"."}`},
{Driver: "Local", MountPath: "/a/d/e", Order: 4, Addition: `{"root_folder_path":"."}`},
{Driver: "Local", MountPath: "/a/d/e1.balance", Order: 4, Addition: `{"root_folder_path":"."}`},
}
for _, storage := range storages {
_, err := op.CreateStorage(context.Background(), storage)

115
internal/op/user.go Normal file
View File

@ -0,0 +1,115 @@
package op
import (
"time"
"github.com/Xhofe/go-cache"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/singleflight"
"github.com/alist-org/alist/v3/pkg/utils"
)
var userCache = cache.NewMemCache(cache.WithShards[*model.User](2))
var userG singleflight.Group[*model.User]
var guestUser *model.User
var adminUser *model.User
func GetAdmin() (*model.User, error) {
if adminUser == nil {
user, err := db.GetUserByRole(model.ADMIN)
if err != nil {
return nil, err
}
adminUser = user
}
return adminUser, nil
}
func GetGuest() (*model.User, error) {
if guestUser == nil {
user, err := db.GetUserByRole(model.GUEST)
if err != nil {
return nil, err
}
guestUser = user
}
return guestUser, nil
}
func GetUserByRole(role int) (*model.User, error) {
return db.GetUserByRole(role)
}
func GetUserByName(username string) (*model.User, error) {
if username == "" {
return nil, errs.EmptyUsername
}
if user, ok := userCache.Get(username); ok {
return user, nil
}
user, err, _ := userG.Do(username, func() (*model.User, error) {
_user, err := db.GetUserByName(username)
if err != nil {
return nil, err
}
userCache.Set(username, _user, cache.WithEx[*model.User](time.Hour))
return _user, nil
})
return user, err
}
func GetUserById(id uint) (*model.User, error) {
return db.GetUserById(id)
}
func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err error) {
return db.GetUsers(pageIndex, pageSize)
}
func CreateUser(u *model.User) error {
u.BasePath = utils.FixAndCleanPath(u.BasePath)
return db.CreateUser(u)
}
func DeleteUserById(id uint) error {
old, err := db.GetUserById(id)
if err != nil {
return err
}
if old.IsAdmin() || old.IsGuest() {
return errs.DeleteAdminOrGuest
}
userCache.Del(old.Username)
return db.DeleteUserById(id)
}
func UpdateUser(u *model.User) error {
old, err := db.GetUserById(u.ID)
if err != nil {
return err
}
if u.IsAdmin() {
adminUser = nil
}
if u.IsGuest() {
guestUser = nil
}
userCache.Del(old.Username)
u.BasePath = utils.FixAndCleanPath(u.BasePath)
return db.UpdateUser(u)
}
func Cancel2FAByUser(u *model.User) error {
u.OtpSecret = ""
return UpdateUser(u)
}
func Cancel2FAById(id uint) error {
user, err := db.GetUserById(id)
if err != nil {
return err
}
return Cancel2FAByUser(user)
}