feat: index enhancement (close #2632 pr #2636)

* feat: index paths as setting

* feat: clear index (#2632)

* feat: check indexMQ more frequently

internal/bootstrap/data/setting.go

@@ -132,6 +132,7 @@ func InitialSettings() []model.SettingItem {
 		// single settings
 		{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
 		{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.INDEX},
+		{Key: conf.IndexPaths, Value: "/", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
 		{Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
 		{Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE},
 	}

internal/conf/const.go

@@ -44,6 +44,7 @@ const (
 
 	// index
 	SearchIndex = "search_index"
+	IndexPaths  = "index_paths"
 	IgnorePaths = "ignore_paths"
 
 	// aria2

internal/search/build.go

@@ -33,10 +33,17 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
 	Quit = make(chan struct{}, 1)
 	indexMQ := mq.NewInMemoryMQ[ObjWithParent]()
 	go func() {
-		ticker := time.NewTicker(5 * time.Second)
+		ticker := time.NewTicker(time.Second)
+		tickCount := 0
 		for {
 			select {
 			case <-ticker.C:
+				tickCount += 1
+				if indexMQ.Len() < 1000 && tickCount != 5 {
+					continue
+				} else if tickCount >= 5 {
+					tickCount = 0
+				}
 				log.Infof("index obj count: %d", objCount)
 				indexMQ.ConsumeAll(func(messages []mq.Message[ObjWithParent]) {
 					if len(messages) != 0 {
@@ -155,6 +162,10 @@ func Update(parent string, objs []model.Obj) {
 	if instance == nil || !instance.Config().AutoUpdate || Running.Load() {
 		return
 	}
+	indexPaths := GetIndexPaths()
+	if !isIndexPath(parent, indexPaths) {
+		return
+	}
 	ignorePaths, err := GetIgnorePaths()
 	if err != nil {
 		log.Errorf("update search index error while get ignore paths: %+v", err)

internal/search/util.go

@@ -35,6 +35,24 @@ func WriteProgress(progress *model.IndexProgress) {
 	}
 }
 
+func GetIndexPaths() []string {
+	indexPaths := make([]string, 0)
+	customIndexPaths := setting.GetStr(conf.IndexPaths)
+	if customIndexPaths != "" {
+		indexPaths = append(indexPaths, strings.Split(customIndexPaths, "\n")...)
+	}
+	return indexPaths
+}
+
+func isIndexPath(path string, indexPaths []string) bool {
+	for _, indexPaths := range indexPaths {
+		if strings.HasPrefix(path, indexPaths) {
+			return true
+		}
+	}
+	return false
+}
+
 func GetIgnorePaths() ([]string, error) {
 	storages, err := db.GetEnabledStorages()
 	if err != nil {