refactor: split the db package hook and cache to the op package (#2747)

* refactor:separate the setting method from the db package to the op package and add the cache

* refactor:separate the meta method from the db package to the op package

* fix:setting not load database data

* refactor:separate the user method from the db package to the op package

* refactor:remove user JoinPath error

* fix:op package user cache

* refactor:fs package list method

* fix:tile virtual paths (close #2743)

* Revert "refactor:remove user JoinPath error"

This reverts commit 4e20daaf9e700da047000d4fd4900abbe05c3848.

* clean path directly may lead to unknown behavior

* fix: The path of the meta passed in must be prefix of reqPath

* chore: rename all virtualPath to mountPath

* fix: `getStoragesByPath` and `GetStorageVirtualFilesByPath`

is_sub_path:

/a/b isn't subpath of /a/bc

* fix: don't save setting if hook error

Co-authored-by: Noah Hsu <i@nn.ci>

server/handles/aria2.go

@@ -3,8 +3,8 @@ package handles
 import (
 	"github.com/alist-org/alist/v3/internal/aria2"
 	"github.com/alist-org/alist/v3/internal/conf"
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/server/common"
 	"github.com/gin-gonic/gin"
 )
@@ -24,7 +24,7 @@ func SetAria2(c *gin.Context) {
 		{Key: conf.Aria2Uri, Value: req.Uri, Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
 		{Key: conf.Aria2Secret, Value: req.Secret, Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
 	}
-	if err := db.SaveSettingItems(items); err != nil {
+	if err := op.SaveSettingItems(items); err != nil {
 		common.ErrorResp(c, err, 500)
 		return
 	}

server/handles/auth.go

@@ -7,8 +7,8 @@ import (
 	"time"
 
 	"github.com/Xhofe/go-cache"
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/server/common"
 	"github.com/gin-gonic/gin"
 	"github.com/pquerna/otp/totp"
@@ -41,7 +41,7 @@ func Login(c *gin.Context) {
 		common.ErrorResp(c, err, 400)
 		return
 	}
-	user, err := db.GetUserByName(req.Username)
+	user, err := op.GetUserByName(req.Username)
 	if err != nil {
 		common.ErrorResp(c, err, 400)
 		loginCache.Set(ip, count+1)
@@ -101,7 +101,7 @@ func UpdateCurrent(c *gin.Context) {
 	if req.Password != "" {
 		user.Password = req.Password
 	}
-	if err := db.UpdateUser(user); err != nil {
+	if err := op.UpdateUser(user); err != nil {
 		common.ErrorResp(c, err, 500)
 	} else {
 		common.SuccessResp(c)
@@ -158,7 +158,7 @@ func Verify2FA(c *gin.Context) {
 		return
 	}
 	user.OtpSecret = req.Secret
-	if err := db.UpdateUser(user); err != nil {
+	if err := op.UpdateUser(user); err != nil {
 		common.ErrorResp(c, err, 500)
 	} else {
 		common.SuccessResp(c)

server/handles/fsmanage.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	stdpath "path"
 
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/errs"
 	"github.com/alist-org/alist/v3/internal/fs"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/internal/sign"
 	"github.com/alist-org/alist/v3/pkg/utils"
 	"github.com/alist-org/alist/v3/server/common"
@@ -32,7 +32,7 @@ func FsMkdir(c *gin.Context) {
 		return
 	}
 	if !user.CanWrite() {
-		meta, err := db.GetNearestMeta(stdpath.Dir(reqPath))
+		meta, err := op.GetNearestMeta(stdpath.Dir(reqPath))
 		if err != nil {
 			if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
 				common.ErrorResp(c, err, 500, true)

server/handles/fsread.go

@@ -6,10 +6,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/errs"
 	"github.com/alist-org/alist/v3/internal/fs"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/internal/sign"
 	"github.com/alist-org/alist/v3/pkg/utils"
 	"github.com/alist-org/alist/v3/server/common"
@@ -61,7 +61,7 @@ func FsList(c *gin.Context) {
 		common.ErrorResp(c, err, 403)
 		return
 	}
-	meta, err := db.GetNearestMeta(reqPath)
+	meta, err := op.GetNearestMeta(reqPath)
 	if err != nil {
 		if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
 			common.ErrorResp(c, err, 500, true)
@@ -118,7 +118,7 @@ func FsDirs(c *gin.Context) {
 		}
 		reqPath = tmp
 	}
-	meta, err := db.GetNearestMeta(reqPath)
+	meta, err := op.GetNearestMeta(reqPath)
 	if err != nil {
 		if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
 			common.ErrorResp(c, err, 500, true)
@@ -233,7 +233,7 @@ func FsGet(c *gin.Context) {
 		common.ErrorResp(c, err, 403)
 		return
 	}
-	meta, err := db.GetNearestMeta(reqPath)
+	meta, err := op.GetNearestMeta(reqPath)
 	if err != nil {
 		if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
 			common.ErrorResp(c, err, 500)
@@ -295,7 +295,7 @@ func FsGet(c *gin.Context) {
 	if err == nil {
 		related = filterRelated(sameLevelFiles, obj)
 	}
-	parentMeta, _ := db.GetNearestMeta(parentPath)
+	parentMeta, _ := op.GetNearestMeta(parentPath)
 	common.SuccessResp(c, FsGetResp{
 		ObjResp: ObjResp{
 			Name:     obj.GetName(),
@@ -344,7 +344,7 @@ func FsOther(c *gin.Context) {
 		common.ErrorResp(c, err, 403)
 		return
 	}
-	meta, err := db.GetNearestMeta(req.Path)
+	meta, err := op.GetNearestMeta(req.Path)
 	if err != nil {
 		if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
 			common.ErrorResp(c, err, 500)

server/handles/meta.go

@@ -6,9 +6,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/model"
-	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/server/common"
 	"github.com/gin-gonic/gin"
 	log "github.com/sirupsen/logrus"
@@ -22,7 +21,7 @@ func ListMetas(c *gin.Context) {
 	}
 	req.Validate()
 	log.Debugf("%+v", req)
-	metas, total, err := db.GetMetas(req.Page, req.PerPage)
+	metas, total, err := op.GetMetas(req.Page, req.PerPage)
 	if err != nil {
 		common.ErrorResp(c, err, 500, true)
 		return
@@ -44,8 +43,7 @@ func CreateMeta(c *gin.Context) {
 		common.ErrorStrResp(c, fmt.Sprintf("%s is illegal: %s", r, err.Error()), 400)
 		return
 	}
-	req.Path = utils.FixAndCleanPath(req.Path)
-	if err := db.CreateMeta(&req); err != nil {
+	if err := op.CreateMeta(&req); err != nil {
 		common.ErrorResp(c, err, 500, true)
 	} else {
 		common.SuccessResp(c)
@@ -63,8 +61,7 @@ func UpdateMeta(c *gin.Context) {
 		common.ErrorStrResp(c, fmt.Sprintf("%s is illegal: %s", r, err.Error()), 400)
 		return
 	}
-	req.Path = utils.FixAndCleanPath(req.Path)
-	if err := db.UpdateMeta(&req); err != nil {
+	if err := op.UpdateMeta(&req); err != nil {
 		common.ErrorResp(c, err, 500, true)
 	} else {
 		common.SuccessResp(c)
@@ -89,7 +86,7 @@ func DeleteMeta(c *gin.Context) {
 		common.ErrorResp(c, err, 400)
 		return
 	}
-	if err := db.DeleteMetaById(uint(id)); err != nil {
+	if err := op.DeleteMetaById(uint(id)); err != nil {
 		common.ErrorResp(c, err, 500, true)
 		return
 	}
@@ -103,7 +100,7 @@ func GetMeta(c *gin.Context) {
 		common.ErrorResp(c, err, 400)
 		return
 	}
-	meta, err := db.GetMetaById(uint(id))
+	meta, err := op.GetMetaById(uint(id))
 	if err != nil {
 		common.ErrorResp(c, err, 500, true)
 		return

server/handles/search.go

@@ -4,9 +4,9 @@ import (
 	"path"
 	"strings"
 
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/errs"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/internal/search"
 	"github.com/alist-org/alist/v3/pkg/utils"
 	"github.com/alist-org/alist/v3/server/common"
@@ -53,7 +53,7 @@ func Search(c *gin.Context) {
 		if !strings.HasPrefix(node.Parent, user.BasePath) {
 			continue
 		}
-		meta, err := db.GetNearestMeta(node.Parent)
+		meta, err := op.GetNearestMeta(node.Parent)
 		if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) {
 			continue
 		}

server/handles/setting.go

@@ -5,8 +5,8 @@ import (
 	"strings"
 
 	"github.com/alist-org/alist/v3/internal/conf"
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/internal/sign"
 	"github.com/alist-org/alist/v3/pkg/utils/random"
 	"github.com/alist-org/alist/v3/server/common"
@@ -17,7 +17,7 @@ import (
 func ResetToken(c *gin.Context) {
 	token := random.Token()
 	item := model.SettingItem{Key: "token", Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}
-	if err := db.SaveSettingItem(item); err != nil {
+	if err := op.SaveSettingItem(&item); err != nil {
 		common.ErrorResp(c, err, 500)
 		return
 	}
@@ -29,14 +29,14 @@ func GetSetting(c *gin.Context) {
 	key := c.Query("key")
 	keys := c.Query("keys")
 	if key != "" {
-		item, err := db.GetSettingItemByKey(key)
+		item, err := op.GetSettingItemByKey(key)
 		if err != nil {
 			common.ErrorResp(c, err, 400)
 			return
 		}
 		common.SuccessResp(c, item)
 	} else {
-		items, err := db.GetSettingItemInKeys(strings.Split(keys, ","))
+		items, err := op.GetSettingItemInKeys(strings.Split(keys, ","))
 		if err != nil {
 			common.ErrorResp(c, err, 400)
 			return
@@ -51,7 +51,7 @@ func SaveSettings(c *gin.Context) {
 		common.ErrorResp(c, err, 400)
 		return
 	}
-	if err := db.SaveSettingItems(req); err != nil {
+	if err := op.SaveSettingItems(req); err != nil {
 		common.ErrorResp(c, err, 500)
 	} else {
 		common.SuccessResp(c)
@@ -65,7 +65,7 @@ func ListSettings(c *gin.Context) {
 	var settings []model.SettingItem
 	var err error
 	if groupsStr == "" && groupStr == "" {
-		settings, err = db.GetSettingItems()
+		settings, err = op.GetSettingItems()
 	} else {
 		var groupStrings []string
 		if groupsStr != "" {
@@ -82,7 +82,7 @@ func ListSettings(c *gin.Context) {
 			}
 			groups = append(groups, group)
 		}
-		settings, err = db.GetSettingItemsInGroups(groups)
+		settings, err = op.GetSettingItemsInGroups(groups)
 	}
 	if err != nil {
 		common.ErrorResp(c, err, 400)
@@ -93,7 +93,7 @@ func ListSettings(c *gin.Context) {
 
 func DeleteSetting(c *gin.Context) {
 	key := c.Query("key")
-	if err := db.DeleteSettingItemByKey(key); err != nil {
+	if err := op.DeleteSettingItemByKey(key); err != nil {
 		common.ErrorResp(c, err, 500)
 		return
 	}
@@ -101,5 +101,5 @@ func DeleteSetting(c *gin.Context) {
 }
 
 func PublicSettings(c *gin.Context) {
-	common.SuccessResp(c, db.GetPublicSettingsMap())
+	common.SuccessResp(c, op.GetPublicSettingsMap())
 }

server/handles/user.go

@@ -3,8 +3,8 @@ package handles
 import (
 	"strconv"
 
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/server/common"
 	"github.com/gin-gonic/gin"
 	log "github.com/sirupsen/logrus"
@@ -18,7 +18,7 @@ func ListUsers(c *gin.Context) {
 	}
 	req.Validate()
 	log.Debugf("%+v", req)
-	users, total, err := db.GetUsers(req.Page, req.PerPage)
+	users, total, err := op.GetUsers(req.Page, req.PerPage)
 	if err != nil {
 		common.ErrorResp(c, err, 500, true)
 		return
@@ -39,7 +39,7 @@ func CreateUser(c *gin.Context) {
 		common.ErrorStrResp(c, "admin or guest user can not be created", 400, true)
 		return
 	}
-	if err := db.CreateUser(&req); err != nil {
+	if err := op.CreateUser(&req); err != nil {
 		common.ErrorResp(c, err, 500, true)
 	} else {
 		common.SuccessResp(c)
@@ -52,7 +52,7 @@ func UpdateUser(c *gin.Context) {
 		common.ErrorResp(c, err, 400)
 		return
 	}
-	user, err := db.GetUserById(req.ID)
+	user, err := op.GetUserById(req.ID)
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
@@ -67,7 +67,7 @@ func UpdateUser(c *gin.Context) {
 	if req.OtpSecret == "" {
 		req.OtpSecret = user.OtpSecret
 	}
-	if err := db.UpdateUser(&req); err != nil {
+	if err := op.UpdateUser(&req); err != nil {
 		common.ErrorResp(c, err, 500)
 	} else {
 		common.SuccessResp(c)
@@ -81,7 +81,7 @@ func DeleteUser(c *gin.Context) {
 		common.ErrorResp(c, err, 400)
 		return
 	}
-	if err := db.DeleteUserById(uint(id)); err != nil {
+	if err := op.DeleteUserById(uint(id)); err != nil {
 		common.ErrorResp(c, err, 500)
 		return
 	}
@@ -95,7 +95,7 @@ func GetUser(c *gin.Context) {
 		common.ErrorResp(c, err, 400)
 		return
 	}
-	user, err := db.GetUserById(uint(id))
+	user, err := op.GetUserById(uint(id))
 	if err != nil {
 		common.ErrorResp(c, err, 500, true)
 		return
@@ -110,7 +110,7 @@ func Cancel2FAById(c *gin.Context) {
 		common.ErrorResp(c, err, 400)
 		return
 	}
-	if err := db.Cancel2FAById(uint(id)); err != nil {
+	if err := op.Cancel2FAById(uint(id)); err != nil {
 		common.ErrorResp(c, err, 500)
 		return
 	}

server/middlewares/auth.go

@@ -2,8 +2,8 @@ package middlewares
 
 import (
 	"github.com/alist-org/alist/v3/internal/conf"
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/internal/setting"
 	"github.com/alist-org/alist/v3/server/common"
 	"github.com/gin-gonic/gin"
@@ -15,7 +15,7 @@ import (
 func Auth(c *gin.Context) {
 	token := c.GetHeader("Authorization")
 	if token == setting.GetStr(conf.Token) {
-		admin, err := db.GetAdmin()
+		admin, err := op.GetAdmin()
 		if err != nil {
 			common.ErrorResp(c, err, 500)
 			c.Abort()
@@ -27,7 +27,7 @@ func Auth(c *gin.Context) {
 		return
 	}
 	if token == "" {
-		guest, err := db.GetGuest()
+		guest, err := op.GetGuest()
 		if err != nil {
 			common.ErrorResp(c, err, 500)
 			c.Abort()
@@ -44,7 +44,7 @@ func Auth(c *gin.Context) {
 		c.Abort()
 		return
 	}
-	user, err := db.GetUserByName(userClaims.Username)
+	user, err := op.GetUserByName(userClaims.Username)
 	if err != nil {
 		common.ErrorResp(c, err, 401)
 		c.Abort()

server/middlewares/down.go

@@ -4,9 +4,9 @@ import (
 	"strings"
 
 	"github.com/alist-org/alist/v3/internal/conf"
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/errs"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/internal/setting"
 	"github.com/alist-org/alist/v3/internal/sign"
 	"github.com/alist-org/alist/v3/pkg/utils"
@@ -18,7 +18,7 @@ import (
 func Down(c *gin.Context) {
 	rawPath := parsePath(c.Param("path"))
 	c.Set("path", rawPath)
-	meta, err := db.GetNearestMeta(rawPath)
+	meta, err := op.GetNearestMeta(rawPath)
 	if err != nil {
 		if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
 			common.ErrorResp(c, err, 500, true)

server/middlewares/fsup.go

@@ -4,9 +4,9 @@ import (
 	"net/url"
 	stdpath "path"
 
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/errs"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/server/common"
 	"github.com/gin-gonic/gin"
 	"github.com/pkg/errors"
@@ -27,7 +27,7 @@ func FsUp(c *gin.Context) {
 		common.ErrorResp(c, err, 403)
 		return
 	}
-	meta, err := db.GetNearestMeta(stdpath.Dir(path))
+	meta, err := op.GetNearestMeta(stdpath.Dir(path))
 	if err != nil {
 		if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
 			common.ErrorResp(c, err, 500, true)

server/webdav.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"net/http"
 
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/pkg/utils"
 	"github.com/alist-org/alist/v3/server/webdav"
 	"github.com/gin-gonic/gin"
@@ -45,7 +45,7 @@ func ServeWebDAV(c *gin.Context) {
 }
 
 func WebDAVAuth(c *gin.Context) {
-	guest, _ := db.GetGuest()
+	guest, _ := op.GetGuest()
 	username, password, ok := c.Request.BasicAuth()
 	if !ok {
 		if c.Request.Method == "OPTIONS" {
@@ -58,7 +58,7 @@ func WebDAVAuth(c *gin.Context) {
 		c.Abort()
 		return
 	}
-	user, err := db.GetUserByName(username)
+	user, err := op.GetUserByName(username)
 	if err != nil || user.ValidatePassword(password) != nil {
 		if c.Request.Method == "OPTIONS" {
 			c.Set("user", guest)

server/webdav/file.go

@@ -10,9 +10,9 @@ import (
 	"path"
 	"path/filepath"
 
-	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/fs"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
 )
 
 // slashClean is equivalent to but slightly more efficient than
@@ -85,7 +85,7 @@ func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn
 	if depth == 1 {
 		depth = 0
 	}
-	meta, _ := db.GetNearestMeta(name)
+	meta, _ := op.GetNearestMeta(name)
 	// Read directory names.
 	objs, err := fs.List(context.WithValue(ctx, "meta", meta), name)
 	//f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0)