feat: multiple search indexes (#2514)

* refactor: abstract search interface

* wip: ~

* fix cycle import

* objs update hook

* wip: ~

* Delete search/none

* auto update index while cache changed

* db searcher

TODO: bleve init issue

cannot open index, metadata missing

* fix size type

why float64??

* fix typo

* fix nil pointer using

* api adapt ui

* bleve: fix clear & change struct

internal/db/db.go

@@ -12,13 +12,18 @@ var db *gorm.DB
 
 func Init(d *gorm.DB) {
 	db = d
-	var err error
-	if conf.Conf.Database.Type == "mysql" {
-		err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
-	} else {
-		err = db.AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
-	}
+	err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode))
 	if err != nil {
 		log.Fatalf("failed migrate database: %s", err.Error())
 	}
 }
+
+func AutoMigrate(dst ...interface{}) error {
+	var err error
+	if conf.Conf.Database.Type == "mysql" {
+		err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(dst...)
+	} else {
+		err = db.AutoMigrate(dst...)
+	}
+	return err
+}

internal/db/searchnode.go

@@ -0,0 +1,49 @@
+package db
+
+import (
+	"fmt"
+
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/pkg/errors"
+)
+
+func CreateSearchNode(node *model.SearchNode) error {
+	return db.Create(node).Error
+}
+
+func DeleteSearchNodesByParent(parent string) error {
+	return db.Where(fmt.Sprintf("%s LIKE ?",
+		columnName("path")), fmt.Sprintf("%s%%", parent)).
+		Delete(&model.SearchNode{}).Error
+}
+
+func ClearSearchNodes() error {
+	return db.Where("1 = 1").Delete(&model.SearchNode{}).Error
+}
+
+func GetSearchNodesByParent(parent string) ([]model.SearchNode, error) {
+	var nodes []model.SearchNode
+	if err := db.Where(fmt.Sprintf("%s = ?",
+		columnName("parent")), parent).Find(&nodes).Error; err != nil {
+		return nil, err
+	}
+	return nodes, nil
+}
+
+func SearchNode(req model.SearchReq) ([]model.SearchNode, int64, error) {
+	searchDB := db.Model(&model.SearchNode{}).Where(
+		fmt.Sprintf("%s LIKE ? AND %s LIKE ?",
+			columnName("parent"),
+			columnName("name")),
+		fmt.Sprintf("%s%%", req.Parent),
+		fmt.Sprintf("%%%s%%", req.Keywords))
+	var count int64
+	if err := searchDB.Count(&count).Error; err != nil {
+		return nil, 0, errors.Wrapf(err, "failed get users count")
+	}
+	var files []model.SearchNode
+	if err := searchDB.Offset((req.Page - 1) * req.PerPage).Limit(req.PerPage).Find(&files).Error; err != nil {
+		return nil, 0, err
+	}
+	return files, count, nil
+}

internal/db/settinghooks.go

@@ -11,81 +11,64 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-type SettingItemHook struct {
-	Hook func(item *model.SettingItem) error
-}
+type SettingItemHook func(item *model.SettingItem) error
 
-var SettingItemHooks = map[string]SettingItemHook{
-	conf.VideoTypes: {
-		Hook: func(item *model.SettingItem) error {
-			conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
-			return nil
-		},
+var settingItemHooks = map[string]SettingItemHook{
+	conf.VideoTypes: func(item *model.SettingItem) error {
+		conf.TypesMap[conf.VideoTypes] = strings.Split(item.Value, ",")
+		return nil
 	},
-	conf.AudioTypes: {
-		Hook: func(item *model.SettingItem) error {
-			conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
-			return nil
-		},
+	conf.AudioTypes: func(item *model.SettingItem) error {
+		conf.TypesMap[conf.AudioTypes] = strings.Split(item.Value, ",")
+		return nil
 	},
-	conf.ImageTypes: {
-		Hook: func(item *model.SettingItem) error {
-			conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
-			return nil
-		},
+	conf.ImageTypes: func(item *model.SettingItem) error {
+		conf.TypesMap[conf.ImageTypes] = strings.Split(item.Value, ",")
+		return nil
 	},
-	conf.TextTypes: {
-		Hook: func(item *model.SettingItem) error {
-			conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
-			return nil
-		},
+	conf.TextTypes: func(item *model.SettingItem) error {
+		conf.TypesMap[conf.TextTypes] = strings.Split(item.Value, ",")
+		return nil
 	},
-	//conf.OfficeTypes: {
-	//	Hook: func(item *model.SettingItem) error {
-	//		conf.TypesMap[conf.OfficeTypes] = strings.Split(item.Value, ",")
-	//		return nil
-	//	},
-	//},
-	conf.ProxyTypes: {
-		func(item *model.SettingItem) error {
-			conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
-			return nil
-		},
+	conf.ProxyTypes: func(item *model.SettingItem) error {
+		conf.TypesMap[conf.ProxyTypes] = strings.Split(item.Value, ",")
+		return nil
 	},
-	conf.PrivacyRegs: {
-		Hook: func(item *model.SettingItem) error {
-			regStrs := strings.Split(item.Value, "\n")
-			regs := make([]*regexp.Regexp, 0, len(regStrs))
-			for _, regStr := range regStrs {
-				reg, err := regexp.Compile(regStr)
-				if err != nil {
-					return errors.WithStack(err)
-				}
-				regs = append(regs, reg)
-			}
-			conf.PrivacyReg = regs
-			return nil
-		},
-	},
-	conf.FilenameCharMapping: {
-		Hook: func(item *model.SettingItem) error {
-			err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
+
+	conf.PrivacyRegs: func(item *model.SettingItem) error {
+		regStrs := strings.Split(item.Value, "\n")
+		regs := make([]*regexp.Regexp, 0, len(regStrs))
+		for _, regStr := range regStrs {
+			reg, err := regexp.Compile(regStr)
 			if err != nil {
-				return err
+				return errors.WithStack(err)
 			}
-			log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
-			return nil
-		},
+			regs = append(regs, reg)
+		}
+		conf.PrivacyReg = regs
+		return nil
+	},
+	conf.FilenameCharMapping: func(item *model.SettingItem) error {
+		err := utils.Json.UnmarshalFromString(item.Value, &conf.FilenameCharMap)
+		if err != nil {
+			return err
+		}
+		log.Debugf("filename char mapping: %+v", conf.FilenameCharMap)
+		return nil
 	},
 }
 
 func HandleSettingItem(item *model.SettingItem) (bool, error) {
-	if hook, ok := SettingItemHooks[item.Key]; ok {
-		return true, hook.Hook(item)
+	if hook, ok := settingItemHooks[item.Key]; ok {
+		return true, hook(item)
 	}
 	return false, nil
 }
 
+func RegisterSettingItemHook(key string, hook SettingItemHook) {
+	settingItemHooks[key] = hook
+}
+
 // func HandleSettingItems(items []model.SettingItem) error {
 // 	for i := range items {
 // 		if err := HandleSettingItem(&items[i]); err != nil {

internal/db/settingitem.go

@@ -108,11 +108,17 @@ func SaveSettingItems(items []model.SettingItem) error {
 			others = append(others, items[i])
 		}
 	}
-	err := db.Save(others).Error
-	if err == nil {
-		settingsUpdate()
+	if len(others) > 0 {
+		err := db.Save(others).Error
+		if err != nil {
+			if len(others) < len(items) {
+				settingsUpdate()
+			}
+			return err
+		}
 	}
-	return err
+	settingsUpdate()
+	return nil
 }
 
 func SaveSettingItem(item model.SettingItem) error {