feat: multiple search indexes (#2514)

* refactor: abstract search interface

* wip: ~

* fix cycle import

* objs update hook

* wip: ~

* Delete search/none

* auto update index while cache changed

* db searcher

TODO: bleve init issue

cannot open index, metadata missing

* fix size type

why float64??

* fix typo

* fix nil pointer using

* api adapt ui

* bleve: fix clear & change struct

internal/bootstrap/data/setting.go

@@ -25,12 +25,13 @@ func initSettings() {
 			settings[i].Flag = model.DEPRECATED
 		}
 	}
-	if settings != nil && len(settings) > 0 {
-		err = db.SaveSettingItems(settings)
-		if err != nil {
-			log.Fatalf("failed save settings: %+v", err)
-		}
-	}
+	// what's going on here???
+	//if settings != nil && len(settings) > 0 {
+	//	err = db.SaveSettingItems(settings)
+	//	if err != nil {
+	//		log.Fatalf("failed save settings: %+v", err)
+	//	}
+	//}
 	// insert new items
 	for i := range initialSettingItems {
 		v := initialSettingItems[i]
@@ -122,11 +123,13 @@ func InitialSettings() []model.SettingItem {
 			Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE},
 		{Key: conf.OcrApi, Value: "https://api.nn.ci/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL},
 		{Key: conf.FilenameCharMapping, Value: `{"/": "|"}`, Type: conf.TypeText, Group: model.GLOBAL},
+		{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.GLOBAL},
 		// aria2 settings
 		{Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
 		{Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE},
 		// single settings
 		{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
+		{Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE},
 	}
 	if flags.Dev {
 		initialSettingItems = append(initialSettingItems, []model.SettingItem{

internal/bootstrap/index.go

@@ -1,10 +1,5 @@
 package bootstrap
 
-import (
-	"github.com/alist-org/alist/v3/internal/conf"
-	"github.com/alist-org/alist/v3/internal/index"
-)
-
 func InitIndex() {
-	index.Init(&conf.Conf.IndexDir)
+	// TODO init ? Probably not.
 }

internal/conf/config.go

@@ -45,13 +45,13 @@ type Config struct {
 	Database       Database  `json:"database"`
 	Scheme         Scheme    `json:"scheme"`
 	TempDir        string    `json:"temp_dir" env:"TEMP_DIR"`
-	IndexDir       string    `json:"index_dir" env:"INDEX_DIR"`
+	BleveDir       string    `json:"bleve_dir" env:"BLEVE_DIR"`
 	Log            LogConfig `json:"log"`
 }
 
 func DefaultConfig() *Config {
 	tempDir := filepath.Join(flags.DataDir, "temp")
-	indexDir := filepath.Join(flags.DataDir, "index")
+	indexDir := filepath.Join(flags.DataDir, "bleve")
 	logPath := filepath.Join(flags.DataDir, "log/log.log")
 	dbPath := filepath.Join(flags.DataDir, "data.db")
 	return &Config{
@@ -66,7 +66,7 @@ func DefaultConfig() *Config {
 			TablePrefix: "x_",
 			DBFile:      dbPath,
 		},
-		IndexDir: indexDir,
+		BleveDir: indexDir,
 		Log: LogConfig{
 			Enable:     true,
 			Name:       logPath,

internal/conf/const.go

@@ -40,13 +40,15 @@ const (
 	PrivacyRegs         = "privacy_regs"
 	OcrApi              = "ocr_api"
 	FilenameCharMapping = "filename_char_mapping"
+	SearchIndex         = "search_index"
 
 	// aria2
 	Aria2Uri    = "aria2_uri"
 	Aria2Secret = "aria2_secret"
 
 	// single
-	Token = "token"
+	Token         = "token"
+	IndexProgress = "index_progress"
 )
 
 const (

internal/db/db.go

@@ -12,13 +12,18 @@ var db *gorm.DB
 
 func Init(d *gorm.DB) {
 	db = d
-	var err error
-	if conf.Conf.Database.Type == "mysql" {
-		err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
-	} else {
-		err = db.AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
-	}
+	err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode))
 	if err != nil {
 		log.Fatalf("failed migrate database: %s", err.Error())
 	}
 }
+
+func AutoMigrate(dst ...interface{}) error {
+	var err error
+	if conf.Conf.Database.Type == "mysql" {
+		err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(dst...)
+	} else {
+		err = db.AutoMigrate(dst...)
+	}
+	return err
+}

internal/db/searchnode.go

@@ -0,0 +1,49 @@
+package db
+
+import (
+	"fmt"
+
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/pkg/errors"
+)
+
+func CreateSearchNode(node *model.SearchNode) error {
+	return db.Create(node).Error
+}
+
+func DeleteSearchNodesByParent(parent string) error {
+	return db.Where(fmt.Sprintf("%s LIKE ?",
+		columnName("path")), fmt.Sprintf("%s%%", parent)).
+		Delete(&model.SearchNode{}).Error
+}
+
+func ClearSearchNodes() error {
+	return db.Where("1 = 1").Delete(&model.SearchNode{}).Error
+}
+
+func GetSearchNodesByParent(parent string) ([]model.SearchNode, error) {
+	var nodes []model.SearchNode
+	if err := db.Where(fmt.Sprintf("%s = ?",
+		columnName("parent")), parent).Find(&nodes).Error; err != nil {
+		return nil, err
+	}
+	return nodes, nil
+}
+
+func SearchNode(req model.SearchReq) ([]model.SearchNode, int64, error) {
+	searchDB := db.Model(&model.SearchNode{}).Where(
+		fmt.Sprintf("%s LIKE ? AND %s LIKE ?",
+			columnName("parent"),
+			columnName("name")),
+		fmt.Sprintf("%s%%", req.Parent),
+		fmt.Sprintf("%%%s%%", req.Keywords))
+	var count int64
+	if err := searchDB.Count(&count).Error; err != nil {
+		return nil, 0, errors.Wrapf(err, "failed get users count")
+	}
+	var files []model.SearchNode
+	if err := searchDB.Offset((req.Page - 1) * req.PerPage).Limit(req.PerPage).Find(&files).Error; err != nil {
+		return nil, 0, err
+	}
+	return files, count, nil
+}

internal/db/settinghooks.go

@@ -11,81 +11,64 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-type SettingItemHook struct {
-	Hook func(item *model.SettingItem) error
-}
+type SettingItemHook func(item *model.SettingItem) error
 
-var SettingItemHooks = map[string]SettingItemHook{
-	conf.VideoTypes: {
-		Hook: func(item *model.SettingItem) error {
-			conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
-			return nil
-		},
+var settingItemHooks = map[string]SettingItemHook{
+	conf.VideoTypes: func(item *model.SettingItem) error {
+		conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
+		return nil
 	},
-	conf.AudioTypes: {
-		Hook: func(item *model.SettingItem) error {
-			conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
-			return nil
-		},
+	conf.AudioTypes: func(item *model.SettingItem) error {
+		conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
+		return nil
 	},
-	conf.ImageTypes: {
-		Hook: func(item *model.SettingItem) error {
-			conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
-			return nil
-		},
+	conf.ImageTypes: func(item *model.SettingItem) error {
+		conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
+		return nil
 	},
-	conf.TextTypes: {
-		Hook: func(item *model.SettingItem) error {
-			conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
-			return nil
-		},
+	conf.TextTypes: func(item *model.SettingItem) error {
+		conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
+		return nil
 	},
-	//conf.OfficeTypes: {
-	//	Hook: func(item *model.SettingItem) error {
-	//		conf.TypesMap[conf.OfficeTypes] = strings.Split(item.Value, ",")
-	//		return nil
-	//	},
-	//},
-	conf.ProxyTypes: {
-		func(item *model.SettingItem) error {
-			conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
-			return nil
-		},
+	conf.ProxyTypes: func(item *model.SettingItem) error {
+		conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
+		return nil
 	},
-	conf.PrivacyRegs: {
-		Hook: func(item *model.SettingItem) error {
-			regStrs := strings.Split(item.Value, "\n")
-			regs := make([]*regexp.Regexp, 0, len(regStrs))
-			for _, regStr := range regStrs {
-				reg, err := regexp.Compile(regStr)
-				if err != nil {
-					return errors.WithStack(err)
-				}
-				regs = append(regs, reg)
-			}
-			conf.PrivacyReg = regs
-			return nil
-		},
-	},
-	conf.FilenameCharMapping: {
-		Hook: func(item *model.SettingItem) error {
-			err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
+
+	conf.PrivacyRegs: func(item *model.SettingItem) error {
+		regStrs := strings.Split(item.Value, "\n")
+		regs := make([]*regexp.Regexp, 0, len(regStrs))
+		for _, regStr := range regStrs {
+			reg, err := regexp.Compile(regStr)
 			if err != nil {
-				return err
+				return errors.WithStack(err)
 			}
-			log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
-			return nil
-		},
+			regs = append(regs, reg)
+		}
+		conf.PrivacyReg = regs
+		return nil
+	},
+	conf.FilenameCharMapping: func(item *model.SettingItem) error {
+		err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
+		if err != nil {
+			return err
+		}
+		log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
+		return nil
 	},
 }
 
 func HandleSettingItem(item *model.SettingItem) (bool, error) {
-	if hook, ok := SettingItemHooks[item.Key]; ok {
-		return true, hook.Hook(item)
+	if hook, ok := settingItemHooks[item.Key]; ok {
+		return true, hook(item)
 	}
 	return false, nil
 }
 
+func RegisterSettingItemHook(key string, hook SettingItemHook) {
+	settingItemHooks[key] = hook
+}
+
 // func HandleSettingItems(items []model.SettingItem) error {
 // 	for i := range items {
 // 		if err := HandleSettingItem(&items[i]); err != nil {

internal/db/settingitem.go

@@ -108,11 +108,17 @@ func SaveSettingItems(items []model.SettingItem) error {
 			others = append(others, items[i])
 		}
 	}
-	err := db.Save(others).Error
-	if err == nil {
-		settingsUpdate()
+	if len(others) > 0 {
+		err := db.Save(others).Error
+		if err != nil {
+			if len(others) < len(items) {
+				settingsUpdate()
+			}
+			return err
+		}
 	}
-	return err
+	settingsUpdate()
+	return nil
 }
 
 func SaveSettingItem(item model.SettingItem) error {

internal/errs/search.go

@@ -0,0 +1,7 @@
+package errs
+
+import "fmt"
+
+var (
+	SearchNotAvailable = fmt.Errorf("search not available")
+)

internal/fs/walk.go

@@ -0,0 +1,45 @@
+package fs
+
+import (
+	"context"
+	"path"
+	"path/filepath"
+
+	"github.com/alist-org/alist/v3/internal/db"
+	"github.com/alist-org/alist/v3/internal/model"
+)
+
+// WalkFS traverses filesystem fs starting at name up to depth levels.
+//
+// WalkFS will stop when current depth > `depth`. For each visited node,
+// WalkFS calls walkFn. If a visited file system node is a directory and
+// walkFn returns path.SkipDir, walkFS will skip traversal of this node.
+func WalkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj, err error) error) error {
+	// This implementation is based on Walk's code in the standard path/path package.
+	walkFnErr := walkFn(name, info, nil)
+	if walkFnErr != nil {
+		if info.IsDir() && walkFnErr == filepath.SkipDir {
+			return nil
+		}
+		return walkFnErr
+	}
+	if !info.IsDir() || depth == 0 {
+		return nil
+	}
+	meta, _ := db.GetNearestMeta(name)
+	// Read directory names.
+	objs, err := List(context.WithValue(ctx, "meta", meta), name)
+	if err != nil {
+		return walkFnErr
+	}
+	for _, fileInfo := range objs {
+		filename := path.Join(name, fileInfo.GetName())
+		if err := WalkFS(ctx, depth-1, filename, fileInfo, walkFn); err != nil {
+			if err == filepath.SkipDir {
+				break
+			}
+			return err
+		}
+	}
+	return nil
+}

internal/index/build.go

@@ -1,102 +0,0 @@
-package index
-
-import (
-	"context"
-	"path"
-	"path/filepath"
-	"time"
-
-	"github.com/alist-org/alist/v3/internal/db"
-	"github.com/alist-org/alist/v3/internal/fs"
-	"github.com/alist-org/alist/v3/internal/model"
-	"github.com/blevesearch/bleve/v2"
-	"github.com/google/uuid"
-)
-
-// walkFS traverses filesystem fs starting at name up to depth levels.
-//
-// walkFS will stop when current depth > `depth`. For each visited node,
-// walkFS calls walkFn. If a visited file system node is a directory and
-// walkFn returns path.SkipDir, walkFS will skip traversal of this node.
-func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj, err error) error) error {
-	// This implementation is based on Walk's code in the standard path/path package.
-	walkFnErr := walkFn(name, info, nil)
-	if walkFnErr != nil {
-		if info.IsDir() && walkFnErr == filepath.SkipDir {
-			return nil
-		}
-		return walkFnErr
-	}
-	if !info.IsDir() || depth == 0 {
-		return nil
-	}
-	meta, _ := db.GetNearestMeta(name)
-	// Read directory names.
-	objs, err := fs.List(context.WithValue(ctx, "meta", meta), name)
-	if err != nil {
-		return walkFnErr
-	}
-	for _, fileInfo := range objs {
-		filename := path.Join(name, fileInfo.GetName())
-		if err := walkFS(ctx, depth-1, filename, fileInfo, walkFn); err != nil {
-			if err == filepath.SkipDir {
-				break
-			}
-			return err
-		}
-	}
-	return nil
-}
-
-type Data struct {
-	Path string
-}
-
-func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int) {
-	// TODO: partial remove indices
-	Reset()
-	var batchs []*bleve.Batch
-	var fileCount uint64 = 0
-	for _, indexPath := range indexPaths {
-		batch := func() *bleve.Batch {
-			batch := index.NewBatch()
-			// TODO: cache unchanged part
-			walkFn := func(indexPath string, info model.Obj, err error) error {
-				for _, avoidPath := range ignorePaths {
-					if indexPath == avoidPath {
-						return filepath.SkipDir
-					}
-				}
-				if !info.IsDir() {
-					batch.Index(uuid.NewString(), Data{Path: indexPath})
-					fileCount += 1
-					if fileCount%100 == 0 {
-						WriteProgress(&Progress{
-							FileCount:    fileCount,
-							IsDone:       false,
-							LastDoneTime: nil,
-						})
-					}
-				}
-				return nil
-			}
-			fi, err := fs.Get(ctx, indexPath)
-			if err != nil {
-				return batch
-			}
-			// TODO: run walkFS concurrently
-			walkFS(ctx, maxDepth, indexPath, fi, walkFn)
-			return batch
-		}()
-		batchs = append(batchs, batch)
-	}
-	for _, batch := range batchs {
-		index.Batch(batch)
-	}
-	now := time.Now()
-	WriteProgress(&Progress{
-		FileCount:    fileCount,
-		IsDone:       true,
-		LastDoneTime: &now,
-	})
-}

internal/index/index.go

@@ -1,47 +0,0 @@
-package index
-
-import (
-	"os"
-
-	"github.com/alist-org/alist/v3/internal/conf"
-	"github.com/blevesearch/bleve/v2"
-	log "github.com/sirupsen/logrus"
-)
-
-var index bleve.Index
-
-func Init(indexPath *string) {
-	fileIndex, err := bleve.Open(*indexPath)
-	if err == bleve.ErrorIndexPathDoesNotExist {
-		log.Infof("Creating new index...")
-		indexMapping := bleve.NewIndexMapping()
-		fileIndex, err = bleve.New(*indexPath, indexMapping)
-		if err != nil {
-			log.Fatal(err)
-		}
-	}
-	index = fileIndex
-	progress := ReadProgress()
-	if !progress.IsDone {
-		log.Warnf("Last index build does not succeed!")
-		WriteProgress(&Progress{
-			FileCount:    progress.FileCount,
-			IsDone:       false,
-			LastDoneTime: nil,
-		})
-	}
-}
-
-func Reset() {
-	log.Infof("Removing old index...")
-	err := os.RemoveAll(conf.Conf.IndexDir)
-	if err != nil {
-		log.Fatal(err)
-	}
-	Init(&conf.Conf.IndexDir)
-	WriteProgress(&Progress{
-		FileCount:    0,
-		IsDone:       false,
-		LastDoneTime: nil,
-	})
-}

internal/index/search.go

@@ -1,19 +0,0 @@
-package index
-
-import (
-	"github.com/blevesearch/bleve/v2"
-	log "github.com/sirupsen/logrus"
-)
-
-func Search(queryString string, size int) (*bleve.SearchResult, error) {
-	query := bleve.NewMatchQuery(queryString)
-	search := bleve.NewSearchRequest(query)
-	search.Size = size
-	search.Fields = []string{"Path"}
-	searchResults, err := index.Search(search)
-	if err != nil {
-		log.Errorf("search error: %+v", err)
-		return nil, err
-	}
-	return searchResults, nil
-}

internal/index/util.go

@@ -1,46 +0,0 @@
-package index
-
-import (
-	"errors"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/alist-org/alist/v3/internal/conf"
-	"github.com/alist-org/alist/v3/pkg/utils"
-	log "github.com/sirupsen/logrus"
-)
-
-type Progress struct {
-	FileCount    uint64     `json:"file_count"`
-	IsDone       bool       `json:"is_done"`
-	LastDoneTime *time.Time `json:"last_done_time"`
-}
-
-func ReadProgress() Progress {
-	progressFilePath := filepath.Join(conf.Conf.IndexDir, "progress.json")
-	_, err := os.Stat(progressFilePath)
-	progress := Progress{0, false, nil}
-	if errors.Is(err, os.ErrNotExist) {
-		if !utils.WriteJsonToFile(progressFilePath, progress) {
-			log.Fatalf("failed to create index progress file")
-		}
-	}
-	progressBytes, err := os.ReadFile(progressFilePath)
-	if err != nil {
-		log.Fatalf("reading index progress file error: %+v", err)
-	}
-	err = utils.Json.Unmarshal(progressBytes, &progress)
-	if err != nil {
-		log.Fatalf("load index progress error: %+v", err)
-	}
-	return progress
-}
-
-func WriteProgress(progress *Progress) {
-	progressFilePath := filepath.Join(conf.Conf.IndexDir, "progress.json")
-	log.Infof("write index progress: %v", progress)
-	if !utils.WriteJsonToFile(progressFilePath, progress) {
-		log.Fatalf("failed to write to index progress file")
-	}
-}

internal/model/req.go

@@ -0,0 +1,20 @@
+package model
+
+type PageReq struct {
+	Page    int `json:"page" form:"page"`
+	PerPage int `json:"per_page" form:"per_page"`
+}
+
+const MaxUint = ^uint(0)
+const MinUint = 0
+const MaxInt = int(MaxUint >> 1)
+const MinInt = -MaxInt - 1
+
+func (p *PageReq) Validate() {
+	if p.Page < 1 {
+		p.Page = 1
+	}
+	if p.PerPage < 1 {
+		p.PerPage = MaxInt
+	}
+}

internal/model/search.go

@@ -0,0 +1,36 @@
+package model
+
+import (
+	"fmt"
+	"time"
+)
+
+type IndexProgress struct {
+	ObjCount     uint64     `json:"obj_count"`
+	IsDone       bool       `json:"is_done"`
+	LastDoneTime *time.Time `json:"last_done_time"`
+	Error        string     `json:"error"`
+}
+
+type SearchReq struct {
+	Parent   string `json:"parent"`
+	Keywords string `json:"keywords"`
+	PageReq
+}
+
+type SearchNode struct {
+	Parent string `json:"parent" gorm:"index"`
+	Name   string `json:"name" gorm:"index"`
+	IsDir  bool   `json:"is_dir"`
+	Size   int64  `json:"size"`
+}
+
+func (p *SearchReq) Validate() error {
+	if p.Page < 1 {
+		return fmt.Errorf("page can't < 1")
+	}
+	if p.PerPage < 1 {
+		return fmt.Errorf("per_page can't < 1")
+	}
+	return nil
+}

internal/op/fs.go

@@ -59,6 +59,12 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to list objs")
 		}
+		// call hooks
+		go func() {
+			for _, hook := range objsUpdateHooks {
+				hook(args.ReqPath, files)
+			}
+		}()
 		if !storage.Config().NoCache {
 			if len(files) > 0 {
 				log.Debugf("set cache: %s => %+v", key, files)

internal/op/hook.go

@@ -0,0 +1,13 @@
+package op
+
+import "github.com/alist-org/alist/v3/internal/model"
+
+type ObjsUpdateHook = func(parent string, objs []model.Obj)
+
+var (
+	objsUpdateHooks = make([]ObjsUpdateHook, 0)
+)
+
+func RegisterObjsUpdateHook(hook ObjsUpdateHook) {
+	objsUpdateHooks = append(objsUpdateHooks, hook)
+}

internal/search/bleve/init.go

@@ -0,0 +1,38 @@
+package bleve
+
+import (
+	"github.com/alist-org/alist/v3/internal/conf"
+	"github.com/alist-org/alist/v3/internal/search/searcher"
+	"github.com/blevesearch/bleve/v2"
+	log "github.com/sirupsen/logrus"
+)
+
+var config = searcher.Config{
+	Name: "bleve",
+}
+
+func Init(indexPath *string) (bleve.Index, error) {
+	log.Debugf("bleve path: %s", *indexPath)
+	fileIndex, err := bleve.Open(*indexPath)
+	if err == bleve.ErrorIndexPathDoesNotExist {
+		log.Infof("Creating new index...")
+		indexMapping := bleve.NewIndexMapping()
+		fileIndex, err = bleve.New(*indexPath, indexMapping)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+	return fileIndex, nil
+}
+
+func init() {
+	searcher.RegisterSearcher(config, func() (searcher.Searcher, error) {
+		b, err := Init(&conf.Conf.BleveDir)
+		if err != nil {
+			return nil, err
+		}
+		return &Bleve{BIndex: b}, nil
+	})
+}

internal/search/bleve/search.go

@@ -0,0 +1,90 @@
+package bleve
+
+import (
+	"context"
+	"os"
+
+	"github.com/alist-org/alist/v3/internal/conf"
+	"github.com/alist-org/alist/v3/internal/errs"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/search/searcher"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/blevesearch/bleve/v2"
+	search2 "github.com/blevesearch/bleve/v2/search"
+	"github.com/google/uuid"
+	log "github.com/sirupsen/logrus"
+)
+
+type Bleve struct {
+	BIndex bleve.Index
+}
+
+func (b *Bleve) Config() searcher.Config {
+	return config
+}
+
+func (b *Bleve) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) {
+	query := bleve.NewMatchQuery(req.Keywords)
+	query.SetField("name")
+	search := bleve.NewSearchRequest(query)
+	search.Size = req.PerPage
+	search.Fields = []string{"*"}
+	searchResults, err := b.BIndex.Search(search)
+	if err != nil {
+		log.Errorf("search error: %+v", err)
+		return nil, 0, err
+	}
+	res, err := utils.SliceConvert(searchResults.Hits, func(src *search2.DocumentMatch) (model.SearchNode, error) {
+		return model.SearchNode{
+			Parent: src.Fields["parent"].(string),
+			Name:   src.Fields["name"].(string),
+			IsDir:  src.Fields["is_dir"].(bool),
+			Size:   int64(src.Fields["size"].(float64)),
+		}, nil
+	})
+	return res, int64(len(res)), nil
+}
+
+func (b *Bleve) Index(ctx context.Context, parent string, obj model.Obj) error {
+	return b.BIndex.Index(uuid.NewString(), model.SearchNode{
+		Parent: parent,
+		Name:   obj.GetName(),
+		IsDir:  obj.IsDir(),
+		Size:   obj.GetSize(),
+	})
+}
+
+func (b *Bleve) Get(ctx context.Context, parent string) ([]model.SearchNode, error) {
+	return nil, errs.NotSupport
+}
+
+func (b *Bleve) Del(ctx context.Context, prefix string) error {
+	return errs.NotSupport
+}
+
+func (b *Bleve) Release(ctx context.Context) error {
+	if b.BIndex != nil {
+		return b.BIndex.Close()
+	}
+	return nil
+}
+
+func (b *Bleve) Clear(ctx context.Context) error {
+	err := b.Release(ctx)
+	if err != nil {
+		return err
+	}
+	log.Infof("Removing old index...")
+	err = os.RemoveAll(conf.Conf.BleveDir)
+	if err != nil {
+		log.Errorf("clear bleve error: %+v", err)
+	}
+	bIndex, err := Init(&conf.Conf.BleveDir)
+	if err != nil {
+		return err
+	}
+	b.BIndex = bIndex
+	return nil
+}
+
+var _ searcher.Searcher = (*Bleve)(nil)

internal/search/build.go

@@ -0,0 +1,100 @@
+package search
+
+import (
+	"context"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/alist-org/alist/v3/internal/db"
+	"github.com/alist-org/alist/v3/internal/fs"
+	"github.com/alist-org/alist/v3/internal/model"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	Running = false
+)
+
+func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int, count bool) error {
+	var objCount uint64 = 0
+	Running = true
+	var (
+		err error
+		fi  model.Obj
+	)
+	defer func() {
+		Running = false
+		now := time.Now()
+		eMsg := ""
+		if err != nil {
+			log.Errorf("build index error: %+v", err)
+			eMsg = err.Error()
+		} else {
+			log.Infof("success build index, count: %d", objCount)
+		}
+		if count {
+			WriteProgress(&model.IndexProgress{
+				ObjCount:     objCount,
+				IsDone:       err == nil,
+				LastDoneTime: &now,
+				Error:        eMsg,
+			})
+		}
+	}()
+	admin, err := db.GetAdmin()
+	if err != nil {
+		return err
+	}
+	if count {
+		WriteProgress(&model.IndexProgress{
+			ObjCount: 0,
+			IsDone:   false,
+		})
+	}
+	for _, indexPath := range indexPaths {
+		walkFn := func(indexPath string, info model.Obj, err error) error {
+			for _, avoidPath := range ignorePaths {
+				if indexPath == avoidPath {
+					return filepath.SkipDir
+				}
+			}
+			// ignore root
+			if indexPath == "/" {
+				return nil
+			}
+			err = instance.Index(ctx, path.Dir(indexPath), info)
+			if err != nil {
+				return err
+			} else {
+				objCount++
+			}
+			if objCount%100 == 0 {
+				log.Infof("index obj count: %d", objCount)
+				log.Debugf("current success index: %s", indexPath)
+				if count {
+					WriteProgress(&model.IndexProgress{
+						ObjCount:     objCount,
+						IsDone:       false,
+						LastDoneTime: nil,
+					})
+				}
+			}
+			return nil
+		}
+		fi, err = fs.Get(ctx, indexPath)
+		if err != nil {
+			return err
+		}
+		// TODO: run walkFS concurrently
+		err = fs.WalkFS(context.WithValue(ctx, "user", admin), maxDepth, indexPath, fi, walkFn)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func Clear(ctx context.Context) error {
+	return instance.Clear(ctx)
+}

internal/search/db/init.go

@@ -0,0 +1,16 @@
+package db
+
+import (
+	"github.com/alist-org/alist/v3/internal/search/searcher"
+)
+
+var config = searcher.Config{
+	Name:       "database",
+	AutoUpdate: true,
+}
+
+func init() {
+	searcher.RegisterSearcher(config, func() (searcher.Searcher, error) {
+		return &DB{}, nil
+	})
+}

internal/search/db/search.go

@@ -0,0 +1,46 @@
+package db
+
+import (
+	"context"
+
+	"github.com/alist-org/alist/v3/internal/db"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/search/searcher"
+)
+
+type DB struct{}
+
+func (D DB) Config() searcher.Config {
+	return config
+}
+
+func (D DB) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) {
+	return db.SearchNode(req)
+}
+
+func (D DB) Index(ctx context.Context, parent string, obj model.Obj) error {
+	return db.CreateSearchNode(&model.SearchNode{
+		Parent: parent,
+		Name:   obj.GetName(),
+		IsDir:  obj.IsDir(),
+		Size:   obj.GetSize(),
+	})
+}
+
+func (D DB) Get(ctx context.Context, parent string) ([]model.SearchNode, error) {
+	return db.GetSearchNodesByParent(parent)
+}
+
+func (D DB) Del(ctx context.Context, prefix string) error {
+	return db.DeleteSearchNodesByParent(prefix)
+}
+
+func (D DB) Release(ctx context.Context) error {
+	return nil
+}
+
+func (D DB) Clear(ctx context.Context) error {
+	return db.ClearSearchNodes()
+}
+
+var _ searcher.Searcher = (*DB)(nil)

internal/search/import.go

@@ -0,0 +1,6 @@
+package search
+
+import (
+	_ "github.com/alist-org/alist/v3/internal/search/bleve"
+	_ "github.com/alist-org/alist/v3/internal/search/db"
+)

internal/search/progress.go

@@ -0,0 +1,36 @@
+package search
+
+import (
+	"context"
+
+	"github.com/alist-org/alist/v3/internal/conf"
+	"github.com/alist-org/alist/v3/internal/db"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/setting"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	log "github.com/sirupsen/logrus"
+)
+
+func Progress(ctx context.Context) (*model.IndexProgress, error) {
+	p := setting.GetStr(conf.IndexProgress)
+	var progress model.IndexProgress
+	err := utils.Json.UnmarshalFromString(p, &progress)
+	return &progress, err
+}
+
+func WriteProgress(progress *model.IndexProgress) {
+	p, err := utils.Json.MarshalToString(progress)
+	if err != nil {
+		log.Errorf("marshal progress error: %+v", err)
+	}
+	err = db.SaveSettingItem(model.SettingItem{
+		Key:   conf.IndexProgress,
+		Value: p,
+		Type:  conf.TypeText,
+		Group: model.SINGLE,
+		Flag:  model.PRIVATE,
+	})
+	if err != nil {
+		log.Errorf("save progress error: %+v", err)
+	}
+}

internal/search/search.go

@@ -0,0 +1,54 @@
+package search
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/alist-org/alist/v3/internal/conf"
+	"github.com/alist-org/alist/v3/internal/db"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/search/searcher"
+	log "github.com/sirupsen/logrus"
+)
+
+var instance searcher.Searcher = nil
+
+// Init or reset index
+func Init(mode string) error {
+	if instance != nil {
+		err := instance.Release(context.Background())
+		if err != nil {
+			log.Errorf("release instance err: %+v", err)
+		}
+		instance = nil
+	}
+	if Running {
+		return fmt.Errorf("index is running")
+	}
+	if mode == "none" {
+		log.Warnf("not enable search")
+		return nil
+	}
+	s, ok := searcher.NewMap[mode]
+	if !ok {
+		return fmt.Errorf("not support index: %s", mode)
+	}
+	i, err := s()
+	if err != nil {
+		log.Errorf("init searcher error: %+v", err)
+	} else {
+		instance = i
+	}
+	return err
+}
+
+func Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) {
+	return instance.Search(ctx, req)
+}
+
+func init() {
+	db.RegisterSettingItemHook(conf.SearchIndex, func(item *model.SettingItem) error {
+		log.Debugf("searcher init, mode: %s", item.Value)
+		return Init(item.Value)
+	})
+}

internal/search/searcher/manage.go

@@ -0,0 +1,9 @@
+package searcher
+
+type New func() (Searcher, error)
+
+var NewMap = map[string]New{}
+
+func RegisterSearcher(config Config, searcher New) {
+	NewMap[config.Name] = searcher
+}

internal/search/searcher/searcher.go

@@ -0,0 +1,29 @@
+package searcher
+
+import (
+	"context"
+
+	"github.com/alist-org/alist/v3/internal/model"
+)
+
+type Config struct {
+	Name       string
+	AutoUpdate bool
+}
+
+type Searcher interface {
+	// Config of the searcher
+	Config() Config
+	// Search specific keywords in specific path
+	Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error)
+	// Index obj with parent
+	Index(ctx context.Context, parent string, obj model.Obj) error
+	// Get by parent
+	Get(ctx context.Context, parent string) ([]model.SearchNode, error)
+	// Del with prefix
+	Del(ctx context.Context, prefix string) error
+	// Release resource
+	Release(ctx context.Context) error
+	// Clear all index
+	Clear(ctx context.Context) error
+}

internal/search/update.go

@@ -0,0 +1,73 @@
+package search
+
+import (
+	"context"
+	"path"
+
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/op"
+	mapset "github.com/deckarep/golang-set/v2"
+	log "github.com/sirupsen/logrus"
+)
+
+func Update(parent string, objs []model.Obj) {
+	if instance != nil && !instance.Config().AutoUpdate {
+		return
+	}
+	ctx := context.Background()
+	// only update when index have built
+	progress, err := Progress(ctx)
+	if err != nil {
+		log.Errorf("update search index error while get progress: %+v", err)
+		return
+	}
+	if !progress.IsDone {
+		return
+	}
+	nodes, err := instance.Get(ctx, parent)
+	if err != nil {
+		log.Errorf("update search index error while get nodes: %+v", err)
+		return
+	}
+	now := mapset.NewSet[string]()
+	for i := range objs {
+		now.Add(objs[i].GetName())
+	}
+	old := mapset.NewSet[string]()
+	for i := range nodes {
+		old.Add(nodes[i].Name)
+	}
+	// delete data that no longer exists
+	toDelete := old.Difference(now)
+	toAdd := now.Difference(old)
+	for i := range nodes {
+		if toDelete.Contains(nodes[i].Name) {
+			err = instance.Del(ctx, path.Join(parent, nodes[i].Name))
+			if err != nil {
+				log.Errorf("update search index error while del old node: %+v", err)
+				return
+			}
+		}
+	}
+	for i := range objs {
+		if toAdd.Contains(objs[i].GetName()) {
+			err = instance.Index(ctx, parent, objs[i])
+			if err != nil {
+				log.Errorf("update search index error while index new node: %+v", err)
+				return
+			}
+			// build index if it's a folder
+			if objs[i].IsDir() {
+				err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, nil, -1, false)
+				if err != nil {
+					log.Errorf("update search index error while build index: %+v", err)
+					return
+				}
+			}
+		}
+	}
+}
+
+func init() {
+	op.RegisterObjsUpdateHook(Update)
+}