feat: optimize index build

internal/search/build.go

@@ -4,16 +4,20 @@ import (
 	"context"
 	"path"
 	"path/filepath"
+	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/alist-org/alist/v3/internal/db"
 	"github.com/alist-org/alist/v3/internal/fs"
 	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/pkg/mq"
+	"github.com/alist-org/alist/v3/pkg/utils"
 	log "github.com/sirupsen/logrus"
 )
 
 var (
-	Running = false
+	Running = atomic.Bool{}
 	Quit    chan struct{}
 )
 
@@ -22,8 +26,9 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
 	if err != nil {
 		return err
 	}
+	var skipDrivers = []string{"AList V2", "AList V3"}
 	for _, storage := range storages {
-		if storage.Driver == "AList V2" || storage.Driver == "AList V3" {
+		if utils.SliceContains(skipDrivers, storage.Driver) {
 			// TODO: request for indexing permission
 			ignorePaths = append(ignorePaths, storage.MountPath)
 		}
@@ -32,66 +37,72 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
 		objCount uint64 = 0
 		fi       model.Obj
 	)
-	Running = true
+	Running.Store(true)
 	Quit = make(chan struct{}, 1)
-	parents := []string{}
-	infos := []model.Obj{}
+	indexMQ := mq.NewInMemoryMQ[ObjWithParent]()
 	go func() {
 		ticker := time.NewTicker(5 * time.Second)
 		for {
 			select {
 			case <-ticker.C:
 				log.Infof("index obj count: %d", objCount)
-				if len(parents) != 0 {
-					log.Debugf("current index: %s", parents[len(parents)-1])
-				}
-				if err = BatchIndex(ctx, parents, infos); err != nil {
-					log.Errorf("build index in batch error: %+v", err)
-				} else {
-					objCount = objCount + uint64(len(parents))
-				}
-				if count {
-					WriteProgress(&model.IndexProgress{
-						ObjCount:     objCount,
-						IsDone:       false,
-						LastDoneTime: nil,
-					})
-				}
-				parents = nil
-				infos = nil
+				indexMQ.ConsumeAll(func(messages []mq.Message[ObjWithParent]) {
+					if len(messages) != 0 {
+						log.Debugf("current index: %s", messages[len(messages)-1].Content.Parent)
+					}
+					if err = BatchIndex(ctx, utils.MustSliceConvert(messages,
+						func(src mq.Message[ObjWithParent]) ObjWithParent {
+							return src.Content
+						})); err != nil {
+						log.Errorf("build index in batch error: %+v", err)
+					} else {
+						objCount = objCount + uint64(len(messages))
+					}
+					if count {
+						WriteProgress(&model.IndexProgress{
+							ObjCount:     objCount,
+							IsDone:       false,
+							LastDoneTime: nil,
+						})
+					}
+				})
+
 			case <-Quit:
-				Running = false
+				Running.Store(false)
 				ticker.Stop()
 				eMsg := ""
 				now := time.Now()
 				originErr := err
-				if err = BatchIndex(ctx, parents, infos); err != nil {
-					log.Errorf("build index in batch error: %+v", err)
-				} else {
-					objCount = objCount + uint64(len(parents))
-				}
-				parents = nil
-				infos = nil
-				if originErr != nil {
-					log.Errorf("build index error: %+v", err)
-					eMsg = err.Error()
-				} else {
-					log.Infof("success build index, count: %d", objCount)
-				}
-				if count {
-					WriteProgress(&model.IndexProgress{
-						ObjCount:     objCount,
-						IsDone:       originErr == nil,
-						LastDoneTime: &now,
-						Error:        eMsg,
-					})
-				}
+				indexMQ.ConsumeAll(func(messages []mq.Message[ObjWithParent]) {
+					if err = BatchIndex(ctx, utils.MustSliceConvert(messages,
+						func(src mq.Message[ObjWithParent]) ObjWithParent {
+							return src.Content
+						})); err != nil {
+						log.Errorf("build index in batch error: %+v", err)
+					} else {
+						objCount = objCount + uint64(len(messages))
+					}
+					if originErr != nil {
+						log.Errorf("build index error: %+v", err)
+						eMsg = err.Error()
+					} else {
+						log.Infof("success build index, count: %d", objCount)
+					}
+					if count {
+						WriteProgress(&model.IndexProgress{
+							ObjCount:     objCount,
+							IsDone:       true,
+							LastDoneTime: &now,
+							Error:        eMsg,
+						})
+					}
+				})
 				return
 			}
 		}
 	}()
 	defer func() {
-		if Running {
+		if Running.Load() {
 			Quit <- struct{}{}
 		}
 	}()
@@ -107,8 +118,11 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
 	}
 	for _, indexPath := range indexPaths {
 		walkFn := func(indexPath string, info model.Obj) error {
+			if !Running.Load() {
+				return filepath.SkipDir
+			}
 			for _, avoidPath := range ignorePaths {
-				if indexPath == avoidPath {
+				if strings.HasPrefix(indexPath, avoidPath) {
 					return filepath.SkipDir
 				}
 			}
@@ -116,8 +130,12 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
 			if indexPath == "/" {
 				return nil
 			}
-			parents = append(parents, path.Dir(indexPath))
-			infos = append(infos, info)
+			indexMQ.Publish(mq.Message[ObjWithParent]{
+				Content: ObjWithParent{
+					Obj:    info,
+					Parent: path.Dir(indexPath),
+				},
+			})
 			return nil
 		}
 		fi, err = fs.Get(ctx, indexPath)

internal/search/search.go

@@ -17,13 +17,17 @@ var instance searcher.Searcher = nil
 // Init or reset index
 func Init(mode string) error {
 	if instance != nil {
+		// unchanged, do nothing
+		if instance.Config().Name == mode {
+			return nil
+		}
 		err := instance.Release(context.Background())
 		if err != nil {
 			log.Errorf("release instance err: %+v", err)
 		}
 		instance = nil
 	}
-	if Running {
+	if Running.Load() {
 		return fmt.Errorf("index is running")
 	}
 	if mode == "none" {
@@ -59,17 +63,22 @@ func Index(ctx context.Context, parent string, obj model.Obj) error {
 	})
 }
 
-func BatchIndex(ctx context.Context, parents []string, objs []model.Obj) error {
+type ObjWithParent struct {
+	Parent string
+	model.Obj
+}
+
+func BatchIndex(ctx context.Context, objs []ObjWithParent) error {
 	if instance == nil {
 		return errs.SearchNotAvailable
 	}
-	if len(parents) == 0 {
+	if len(objs) == 0 {
 		return nil
 	}
-	searchNodes := []model.SearchNode{}
-	for i := range parents {
+	var searchNodes []model.SearchNode
+	for i := range objs {
 		searchNodes = append(searchNodes, model.SearchNode{
-			Parent: parents[i],
+			Parent: objs[i].Parent,
 			Name:   objs[i].GetName(),
 			IsDir:  objs[i].IsDir(),
 			Size:   objs[i].GetSize(),

internal/search/update.go

@@ -11,7 +11,7 @@ import (
 )
 
 func Update(parent string, objs []model.Obj) {
-	if instance != nil && !instance.Config().AutoUpdate {
+	if instance == nil || !instance.Config().AutoUpdate || Running.Load() {
 		return
 	}
 	ctx := context.Background()