ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: split the db package hook and cache to the op package (#2747)

Browse Source 
* refactor:separate the setting method from the db package to the op package and add the cache

* refactor:separate the meta method from the db package to the op package

* fix:setting not load database data

* refactor:separate the user method from the db package to the op package

* refactor:remove user JoinPath error

* fix:op package user cache

* refactor:fs package list method

* fix:tile virtual paths (close #2743)

* Revert "refactor:remove user JoinPath error"

This reverts commit 4e20daaf9e700da047000d4fd4900abbe05c3848.

* clean path directly may lead to unknown behavior

* fix: The path of the meta passed in must be prefix of reqPath

* chore: rename all virtualPath to mountPath

* fix: `getStoragesByPath` and `GetStorageVirtualFilesByPath`

is_sub_path:

/a/b isn't subpath of /a/bc

* fix: don't save setting if hook error

Co-authored-by: Noah Hsu <i@nn.ci>
This commit is contained in:
foxxorcat
2022-12-18 19:51:20 +08:00
committed by GitHub
parent f38f4f401b
commit 6024e8d832
39 changed files with 741 additions and 543 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
internal/db/meta.go
View File

@ -1,52 +1,16 @@
package db
import (
stdpath "path"
"time"
"github.com/Xhofe/go-cache"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/singleflight"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
"gorm.io/gorm"
)
var metaCache = cache.NewMemCache(cache.WithShards[*model.Meta](2))
// metaG maybe not needed
var metaG singleflight.Group[*model.Meta]
func GetNearestMeta(path string) (*model.Meta, error) {
path = utils.FixAndCleanPath(path)
meta, err := GetMetaByPath(path)
if err == nil {
return meta, nil
}
if errors.Cause(err) != gorm.ErrRecordNotFound {
return nil, err
}
if path == "/" {
return nil, errors.WithStack(errs.MetaNotFound)
}
return GetNearestMeta(stdpath.Dir(path))
}
func GetMetaByPath(path string) (*model.Meta, error) {
meta, ok := metaCache.Get(path)
if ok {
return meta, nil
meta := model.Meta{Path: path}
if err := db.Where(meta).First(&meta).Error; err != nil {
return nil, errors.Wrapf(err, "failed select meta")
}
meta, err, _ := metaG.Do(path, func() (*model.Meta, error) {
meta := model.Meta{Path: path}
if err := db.Where(meta).First(&meta).Error; err != nil {
return nil, errors.Wrapf(err, "failed select meta")
}
metaCache.Set(path, &meta, cache.WithEx[*model.Meta](time.Hour))
return &meta, nil
})
return meta, err
return &meta, nil
}
func GetMetaById(id uint) (*model.Meta, error) {
@ -62,32 +26,20 @@ func CreateMeta(u *model.Meta) error {
}
func UpdateMeta(u *model.Meta) error {
old, err := GetMetaById(u.ID)
if err != nil {
return err
}
metaCache.Del(old.Path)
return errors.WithStack(db.Save(u).Error)
}
func GetMetas(pageIndex, pageSize int) ([]model.Meta, int64, error) {
func GetMetas(pageIndex, pageSize int) (metas []model.Meta, count int64, err error) {
metaDB := db.Model(&model.Meta{})
var count int64
if err := metaDB.Count(&count).Error; err != nil {
if err = metaDB.Count(&count).Error; err != nil {
return nil, 0, errors.Wrapf(err, "failed get metas count")
}
var metas []model.Meta
if err := metaDB.Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&metas).Error; err != nil {
if err = metaDB.Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&metas).Error; err != nil {
return nil, 0, errors.Wrapf(err, "failed get find metas")
}
return metas, count, nil
}
func DeleteMetaById(id uint) error {
old, err := GetMetaById(id)
if err != nil {
return err
}
metaCache.Del(old.Path)
return errors.WithStack(db.Delete(&model.Meta{}, id).Error)
}

60
internal/db/meta_test.go
View File

@ -1,60 +0,0 @@
package db
import (
"testing"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/pkg/errors"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
func init() {
db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
if err != nil {
panic("failed to connect database")
}
Init(db)
}
func TestCreateMeta(t *testing.T) {
metas := []model.Meta{
{Path: "/"},
{Path: "/a"},
{Path: "/a/b"},
{Path: "/a/b/c"},
}
for _, meta := range metas {
err := CreateMeta(&meta)
if err != nil {
t.Errorf("failed to create meta: %+v", err)
}
}
}
func TestUpdateMeta(t *testing.T) {
meta := model.Meta{ID: 1, Path: "/b"}
err := UpdateMeta(&meta)
if err != nil {
t.Errorf("failed to update meta: %+v", err)
}
}
func TestGetNearestMeta1(t *testing.T) {
meta, err := GetNearestMeta("/b/c/d")
if err != nil {
t.Errorf("failed to get nearest meta: %+v", err)
}
if meta.Path != "/b" {
t.Errorf("unexpected meta: %+v", meta)
}
}
func TestGetNearestMeta2(t *testing.T) {
meta, err := GetNearestMeta("/c/d/e")
if errors.Cause(err) != errs.MetaNotFound {
t.Errorf("unexpected error: %+v", err)
t.Errorf("unexpected meta: %+v", meta)
}
}

79
internal/db/settinghooks.go
View File

@ -1,79 +0,0 @@
package db
import (
"regexp"
"strings"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
type SettingItemHook func(item *model.SettingItem) error
var settingItemHooks = map[string]SettingItemHook{
conf.VideoTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
return nil
},
conf.AudioTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ImageTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
return nil
},
conf.TextTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
return nil
},
conf.ProxyTypes: func(item *model.SettingItem) error {
conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
return nil
},
conf.PrivacyRegs: func(item *model.SettingItem) error {
regStrs := strings.Split(item.Value, "\n")
regs := make([]*regexp.Regexp, 0, len(regStrs))
for _, regStr := range regStrs {
reg, err := regexp.Compile(regStr)
if err != nil {
return errors.WithStack(err)
}
regs = append(regs, reg)
}
conf.PrivacyReg = regs
return nil
},
conf.FilenameCharMapping: func(item *model.SettingItem) error {
err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
if err != nil {
return err
}
log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
return nil
},
}
func HandleSettingItem(item *model.SettingItem) (bool, error) {
if hook, ok := settingItemHooks[item.Key]; ok {
return true, hook(item)
}
return false, nil
}
func RegisterSettingItemHook(key string, hook SettingItemHook) {
settingItemHooks[key] = hook
}
// func HandleSettingItems(items []model.SettingItem) error {
// for i := range items {
// if err := HandleSettingItem(&items[i]); err != nil {
// return err
// }
// }
// return nil
// }

106
internal/db/settingitem.go
View File

@ -4,45 +4,9 @@ import (
"fmt"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/generic_sync"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
var settingsMap generic_sync.MapOf[string, string]
var publicSettingsMap generic_sync.MapOf[string, string]
func settingsUpdate() {
settingsMap.Clear()
publicSettingsMap.Clear()
}
func GetPublicSettingsMap() map[string]string {
if publicSettingsMap.Empty() {
settingItems, err := GetPublicSettingItems()
if err != nil {
log.Errorf("failed to get settingItems: %+v", err)
}
for _, settingItem := range settingItems {
publicSettingsMap.Store(settingItem.Key, settingItem.Value)
}
}
return publicSettingsMap.ToMap()
}
func GetSettingsMap() *generic_sync.MapOf[string, string] {
if settingsMap.Empty() {
settingItems, err := GetSettingItems()
if err != nil {
log.Errorf("failed to get settingItems: %+v", err)
}
for _, settingItem := range settingItems {
settingsMap.Store(settingItem.Key, settingItem.Value)
}
}
return &settingsMap
}
func GetSettingItems() ([]model.SettingItem, error) {
var settingItems []model.SettingItem
if err := db.Find(&settingItems).Error; err != nil {
@ -59,18 +23,18 @@ func GetSettingItemByKey(key string) (*model.SettingItem, error) {
return &settingItem, nil
}
func GetSettingItemInKeys(keys []string) ([]model.SettingItem, error) {
var settingItem []model.SettingItem
if err := db.Where(fmt.Sprintf("%s in ?", columnName("key")), keys).Find(&settingItem).Error; err != nil {
return nil, errors.WithStack(err)
}
return settingItem, nil
}
// func GetSettingItemInKeys(keys []string) ([]model.SettingItem, error) {
// var settingItem []model.SettingItem
// if err := db.Where(fmt.Sprintf("%s in ?", columnName("key")), keys).Find(&settingItem).Error; err != nil {
// return nil, errors.WithStack(err)
// }
// return settingItem, nil
// }
func GetPublicSettingItems() ([]model.SettingItem, error) {
var settingItems []model.SettingItem
if err := db.Where(fmt.Sprintf("%s in ?", columnName("flag")), []int{model.PUBLIC, model.READONLY}).Find(&settingItems).Error; err != nil {
return nil, err
return nil, errors.WithStack(err)
}
return settingItems, nil
}
@ -91,58 +55,14 @@ func GetSettingItemsInGroups(groups []int) ([]model.SettingItem, error) {
return settingItems, nil
}
func SaveSettingItems(items []model.SettingItem) error {
others := make([]model.SettingItem, 0)
for i := range items {
if ok, err := HandleSettingItem(&items[i]); ok {
if err != nil {
return err
} else {
err = db.Save(items[i]).Error
if err != nil {
return errors.WithStack(err)
}
}
} else {
others = append(others, items[i])
}
}
if len(others) > 0 {
err := db.Save(others).Error
if err != nil {
if len(others) < len(items) {
settingsUpdate()
}
return err
}
}
settingsUpdate()
return nil
func SaveSettingItems(items []model.SettingItem) (err error) {
return errors.WithStack(db.Save(items).Error)
}
func SaveSettingItem(item model.SettingItem) error {
_, err := HandleSettingItem(&item)
if err != nil {
return err
}
err = db.Save(item).Error
if err == nil {
settingsUpdate()
}
return errors.WithStack(err)
func SaveSettingItem(item *model.SettingItem) error {
return errors.WithStack(db.Save(item).Error)
}
func DeleteSettingItemByKey(key string) error {
settingItem := model.SettingItem{
Key: key,
}
old, err := GetSettingItemByKey(key)
if err != nil {
return errors.WithMessage(err, "failed to get settingItem")
}
if !old.IsDeprecated() {
return errors.Errorf("setting [%s] is not deprecated", key)
}
settingsUpdate()
return errors.WithStack(db.Delete(&settingItem).Error)
return errors.WithStack(db.Delete(&model.SettingItem{Key: key}).Error)
}

85
internal/db/user.go
View File

@ -1,61 +1,24 @@
package db
import (
"time"
"github.com/Xhofe/go-cache"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/singleflight"
"github.com/pkg/errors"
)
var userCache = cache.NewMemCache(cache.WithShards[*model.User](2))
var userG singleflight.Group[*model.User]
var guest *model.User
var admin *model.User
func GetAdmin() (*model.User, error) {
if admin != nil {
return admin, nil
}
user := model.User{Role: model.ADMIN}
func GetUserByRole(role int) (*model.User, error) {
user := model.User{Role: role}
if err := db.Where(user).Take(&user).Error; err != nil {
return nil, err
}
admin = &user
return &user, nil
}
func GetGuest() (*model.User, error) {
if guest != nil {
return guest, nil
}
user := model.User{Role: model.GUEST}
if err := db.Where(user).Take(&user).Error; err != nil {
return nil, err
}
guest = &user
return &user, nil
}
func GetUserByName(username string) (*model.User, error) {
if username == "" {
return nil, errors.WithStack(errs.EmptyUsername)
user := model.User{Username: username}
if err := db.Where(user).First(&user).Error; err != nil {
return nil, errors.Wrapf(err, "failed find user")
}
user, ok := userCache.Get(username)
if ok {
return user, nil
}
user, err, _ := userG.Do(username, func() (*model.User, error) {
user := model.User{Username: username}
if err := db.Where(user).First(&user).Error; err != nil {
return nil, errors.Wrapf(err, "failed find user")
}
userCache.Set(username, &user, cache.WithEx[*model.User](time.Hour))
return &user, nil
})
return user, err
return &user, nil
}
func GetUserById(id uint) (*model.User, error) {
@ -71,40 +34,14 @@ func CreateUser(u *model.User) error {
}
func UpdateUser(u *model.User) error {
old, err := GetUserById(u.ID)
if err != nil {
return err
}
userCache.Del(old.Username)
if u.IsGuest() {
guest = nil
}
if u.IsAdmin() {
admin = nil
}
return errors.WithStack(db.Save(u).Error)
}
func Cancel2FAByUser(u *model.User) error {
u.OtpSecret = ""
return errors.WithStack(UpdateUser(u))
}
func Cancel2FAById(id uint) error {
user, err := GetUserById(id)
if err != nil {
return err
}
return Cancel2FAByUser(user)
}
func GetUsers(pageIndex, pageSize int) ([]model.User, int64, error) {
func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err error) {
userDB := db.Model(&model.User{})
var count int64
if err := userDB.Count(&count).Error; err != nil {
return nil, 0, errors.Wrapf(err, "failed get users count")
}
var users []model.User
if err := userDB.Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&users).Error; err != nil {
return nil, 0, errors.Wrapf(err, "failed get find users")
}
@ -112,13 +49,5 @@ func GetUsers(pageIndex, pageSize int) ([]model.User, int64, error) {
}
func DeleteUserById(id uint) error {
old, err := GetUserById(id)
if err != nil {
return err
}
if old.IsAdmin() || old.IsGuest() {
return errors.WithStack(errs.DeleteAdminOrGuest)
}
userCache.Del(old.Username)
return errors.WithStack(db.Delete(&model.User{}, id).Error)
}