From 6f1aeb47fdaa6faedd1672a2a4eea21cbc172d23 Mon Sep 17 00:00:00 2001 From: BoYanZh Date: Fri, 9 Dec 2022 10:02:13 +0800 Subject: [PATCH] feat: index enhancement (close #2632 pr #2636) * feat: index paths as setting * feat: clear index (#2632) * feat: check indexMQ more frequently --- internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 1 + internal/search/build.go | 13 ++++++++++++- internal/search/util.go | 18 ++++++++++++++++++ pkg/mq/mq.go | 5 +++++ server/handles/index.go | 26 +++++++++++++++++++++++++- server/router.go | 1 + 7 files changed, 63 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 20269a21..3a18f2c0 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -132,6 +132,7 @@ func InitialSettings() []model.SettingItem { // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.INDEX}, + {Key: conf.IndexPaths, Value: "/", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`}, {Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`}, {Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE}, } diff --git a/internal/conf/const.go b/internal/conf/const.go index 5da67c98..0830ff2f 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -44,6 +44,7 @@ const ( // index SearchIndex = "search_index" + IndexPaths = "index_paths" IgnorePaths = "ignore_paths" // aria2 diff --git a/internal/search/build.go b/internal/search/build.go index 0ab519a0..d4f8d172 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -33,10 +33,17 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth Quit = make(chan struct{}, 1) indexMQ := mq.NewInMemoryMQ[ObjWithParent]() go func() { - ticker := time.NewTicker(5 * time.Second) + ticker := time.NewTicker(time.Second) + tickCount := 0 for { select { case <-ticker.C: + tickCount += 1 + if indexMQ.Len() < 1000 && tickCount != 5 { + continue + } else if tickCount >= 5 { + tickCount = 0 + } log.Infof("index obj count: %d", objCount) indexMQ.ConsumeAll(func(messages []mq.Message[ObjWithParent]) { if len(messages) != 0 { @@ -155,6 +162,10 @@ func Update(parent string, objs []model.Obj) { if instance == nil || !instance.Config().AutoUpdate || Running.Load() { return } + indexPaths := GetIndexPaths() + if !isIndexPath(parent, indexPaths) { + return + } ignorePaths, err := GetIgnorePaths() if err != nil { log.Errorf("update search index error while get ignore paths: %+v", err) diff --git a/internal/search/util.go b/internal/search/util.go index 75767285..c2ebd61b 100644 --- a/internal/search/util.go +++ b/internal/search/util.go @@ -35,6 +35,24 @@ func WriteProgress(progress *model.IndexProgress) { } } +func GetIndexPaths() []string { + indexPaths := make([]string, 0) + customIndexPaths := setting.GetStr(conf.IndexPaths) + if customIndexPaths != "" { + indexPaths = append(indexPaths, strings.Split(customIndexPaths, "\n")...) + } + return indexPaths +} + +func isIndexPath(path string, indexPaths []string) bool { + for _, indexPaths := range indexPaths { + if strings.HasPrefix(path, indexPaths) { + return true + } + } + return false +} + func GetIgnorePaths() ([]string, error) { storages, err := db.GetEnabledStorages() if err != nil { diff --git a/pkg/mq/mq.go b/pkg/mq/mq.go index 4c2176ae..35f5a159 100644 --- a/pkg/mq/mq.go +++ b/pkg/mq/mq.go @@ -18,6 +18,7 @@ type MQ[T any] interface { Consume(BasicConsumer[T]) ConsumeAll(AllConsumer[T]) Clear() + Len() int } type inMemoryMQ[T any] struct { @@ -54,3 +55,7 @@ func (mq *inMemoryMQ[T]) Clear() { defer mq.Unlock() mq.queue.Clear() } + +func (mq *inMemoryMQ[T]) Len() int { + return mq.queue.Len() +} diff --git a/server/handles/index.go b/server/handles/index.go index 5c781066..0c0793a2 100644 --- a/server/handles/index.go +++ b/server/handles/index.go @@ -3,8 +3,10 @@ package handles import ( "context" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/search" "github.com/alist-org/alist/v3/server/common" + mapset "github.com/deckarep/golang-set/v2" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) @@ -26,6 +28,13 @@ func BuildIndex(c *gin.Context) { common.ErrorStrResp(c, "index is running", 400) return } + indexPaths := search.GetIndexPaths() + indexPaths = append(indexPaths, req.Paths...) + indexPathsSet := mapset.NewSet[string]() + for _, indexPath := range indexPaths { + indexPathsSet.Add(indexPath) + } + indexPaths = indexPathsSet.ToSlice() ignorePaths, err := search.GetIgnorePaths() if err != nil { common.ErrorResp(c, err, 500) @@ -50,7 +59,7 @@ func BuildIndex(c *gin.Context) { } } } - err = search.BuildIndex(context.Background(), req.Paths, ignorePaths, req.MaxDepth, true) + err = search.BuildIndex(context.Background(), indexPaths, ignorePaths, req.MaxDepth, true) if err != nil { log.Errorf("build index error: %+v", err) } @@ -67,6 +76,21 @@ func StopIndex(c *gin.Context) { common.SuccessResp(c) } +func ClearIndex(c *gin.Context) { + if search.Running.Load() { + common.ErrorStrResp(c, "index is running", 400) + return + } + search.Clear(c) + search.WriteProgress(&model.IndexProgress{ + ObjCount: 0, + IsDone: false, + LastDoneTime: nil, + Error: "", + }) + common.SuccessResp(c) +} + func GetProgress(c *gin.Context) { progress, err := search.Progress() if err != nil { diff --git a/server/router.go b/server/router.go index d81ebc65..84246792 100644 --- a/server/router.go +++ b/server/router.go @@ -111,6 +111,7 @@ func admin(g *gin.RouterGroup) { index := g.Group("/index") index.POST("/build", middlewares.SearchIndex, handles.BuildIndex) index.POST("/stop", middlewares.SearchIndex, handles.StopIndex) + index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex) index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) }