From 26fe0a768423b4ff576fb2610ba90da9799fbc4a Mon Sep 17 00:00:00 2001 From: Noah Hsu Date: Tue, 17 Jan 2023 17:33:18 +0800 Subject: [PATCH] feat: customize index max depth Because some driver's issue may cause infinite loop --- internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 2 +- internal/db/searchnode.go | 10 +++++----- internal/search/build.go | 5 +++-- internal/search/db/search.go | 4 ++-- internal/search/db_non_full_text/search.go | 4 ++-- server/handles/index.go | 14 +++++--------- 7 files changed, 19 insertions(+), 21 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index f6134884..24834ac5 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -146,6 +146,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,none", Group: model.INDEX}, {Key: conf.AutoUpdateIndex, Value: "false", Type: conf.TypeBool, Group: model.INDEX}, {Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`}, + {Key: conf.MaxIndexDepth, Value: "20", Type: conf.TypeNumber, Group: model.INDEX, Flag: model.PRIVATE, Help: `max depth of index`}, {Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE}, // GitHub settings diff --git a/internal/conf/const.go b/internal/conf/const.go index 6a75634e..7ebb3e3b 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -44,8 +44,8 @@ const ( // index SearchIndex = "search_index" AutoUpdateIndex = "auto_update_index" - IndexPaths = "index_paths" IgnorePaths = "ignore_paths" + MaxIndexDepth = "max_index_depth" // aria2 Aria2Uri = "aria2_uri" diff --git a/internal/db/searchnode.go b/internal/db/searchnode.go index 909d6452..2e5b1094 100644 --- a/internal/db/searchnode.go +++ b/internal/db/searchnode.go @@ -2,7 +2,7 @@ package db import ( "fmt" - "path" + stdpath "path" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -29,13 +29,13 @@ func BatchCreateSearchNodes(nodes *[]model.SearchNode) error { return db.CreateInBatches(nodes, 1000).Error } -func DeleteSearchNodesByParent(prefix string) error { - prefix = utils.FixAndCleanPath(prefix) - err := db.Where(whereInParent(prefix)).Delete(&model.SearchNode{}).Error +func DeleteSearchNodesByParent(path string) error { + path = utils.FixAndCleanPath(path) + err := db.Where(whereInParent(path)).Delete(&model.SearchNode{}).Error if err != nil { return err } - dir, name := path.Split(prefix) + dir, name := stdpath.Split(path) return db.Where(fmt.Sprintf("%s = ? AND %s = ?", columnName("parent"), columnName("name")), dir, name).Delete(&model.SearchNode{}).Error diff --git a/internal/search/build.go b/internal/search/build.go index 399653a9..52b6be95 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -217,10 +217,11 @@ func Update(parent string, objs []model.Obj) { } // build index if it's a folder if objs[i].IsDir() { + dir := path.Join(parent, objs[i].GetName()) err = BuildIndex(ctx, - []string{path.Join(parent, objs[i].GetName())}, + []string{dir}, conf.SlicesMap[conf.IgnorePaths], - -1, false) + setting.GetInt(conf.MaxIndexDepth, 20)-strings.Count(dir, "/"), false) if err != nil { log.Errorf("update search index error while build index: %+v", err) return diff --git a/internal/search/db/search.go b/internal/search/db/search.go index b6613e3b..70baef06 100644 --- a/internal/search/db/search.go +++ b/internal/search/db/search.go @@ -30,8 +30,8 @@ func (D DB) Get(ctx context.Context, parent string) ([]model.SearchNode, error) return db.GetSearchNodesByParent(parent) } -func (D DB) Del(ctx context.Context, prefix string) error { - return db.DeleteSearchNodesByParent(prefix) +func (D DB) Del(ctx context.Context, path string) error { + return db.DeleteSearchNodesByParent(path) } func (D DB) Release(ctx context.Context) error { diff --git a/internal/search/db_non_full_text/search.go b/internal/search/db_non_full_text/search.go index 32d8c492..8589114b 100644 --- a/internal/search/db_non_full_text/search.go +++ b/internal/search/db_non_full_text/search.go @@ -30,8 +30,8 @@ func (D DB) Get(ctx context.Context, parent string) ([]model.SearchNode, error) return db.GetSearchNodesByParent(parent) } -func (D DB) Del(ctx context.Context, prefix string) error { - return db.DeleteSearchNodesByParent(prefix) +func (D DB) Del(ctx context.Context, path string) error { + return db.DeleteSearchNodesByParent(path) } func (D DB) Release(ctx context.Context) error { diff --git a/server/handles/index.go b/server/handles/index.go index 754e1fea..693b67c0 100644 --- a/server/handles/index.go +++ b/server/handles/index.go @@ -6,23 +6,19 @@ import ( "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/search" + "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) -type BuildOrUpdateIndexReq struct { +type UpdateIndexReq struct { Paths []string `json:"paths"` MaxDepth int `json:"max_depth"` //IgnorePaths []string `json:"ignore_paths"` } func BuildIndex(c *gin.Context) { - var req BuildOrUpdateIndexReq - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } if search.Running.Load() { common.ErrorStrResp(c, "index is running", 400) return @@ -34,8 +30,8 @@ func BuildIndex(c *gin.Context) { log.Errorf("clear index error: %+v", err) return } - err = search.BuildIndex(context.Background(), req.Paths, - conf.SlicesMap[conf.IgnorePaths], req.MaxDepth, true) + err = search.BuildIndex(context.Background(), []string{"/"}, + conf.SlicesMap[conf.IgnorePaths], setting.GetInt(conf.MaxIndexDepth, 20), true) if err != nil { log.Errorf("build index error: %+v", err) } @@ -44,7 +40,7 @@ func BuildIndex(c *gin.Context) { } func UpdateIndex(c *gin.Context) { - var req BuildOrUpdateIndexReq + var req UpdateIndexReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return