ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: update index by req.Paths

Browse Source
This commit is contained in:
Noah Hsu
2022-12-24 20:23:04 +08:00
parent 5e28d0f96a
commit e118f4a3b9
7 changed files with 99 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/bootstrap/data/setting.go
View File

@ -143,7 +143,6 @@ func InitialSettings() []model.SettingItem {
// single settings
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,bleve,none", Group: model.INDEX},
{Key: conf.IndexPaths, Value: "/", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
{Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
{Key: conf.IndexProgress, Value: "{}", Type: conf.TypeText, Group: model.SINGLE, Flag: model.PRIVATE},
}

24
internal/op/hook.go
View File

@ -5,6 +5,7 @@ import (
"strings"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
@ -90,12 +91,17 @@ func HandleSettingItemHook(item *model.SettingItem) (hasHook bool, err error) {
return false, nil
}
//func HandleSettingItemsHook(items []model.SettingItem) (err error) {
// for i := 0; i < len(items); i++ {
// _, err = HandleSettingItemHook(&items[i])
// if err != nil {
// return err
// }
// }
// return nil
//}
// Storage
type StorageHook func(typ string, storage driver.Driver)
var storageHooks = make([]StorageHook, 0)
func callStorageHooks(typ string, storage driver.Driver) {
for _, hook := range storageHooks {
hook(typ, storage)
}
}
func RegisterStorageHook(hook StorageHook) {
storageHooks = append(storageHooks, hook)
}

17
internal/op/storage.go
View File

@ -29,11 +29,11 @@ func HasStorage(mountPath string) bool {
return storagesMap.Has(utils.FixAndCleanPath(mountPath))
}
func GetStorageByMountPath(virtualPath string) (driver.Driver, error) {
virtualPath = utils.FixAndCleanPath(virtualPath)
storageDriver, ok := storagesMap.Load(virtualPath)
func GetStorageByMountPath(mountPath string) (driver.Driver, error) {
mountPath = utils.FixAndCleanPath(mountPath)
storageDriver, ok := storagesMap.Load(mountPath)
if !ok {
return nil, errors.Errorf("no mount path for an storage is: %s", virtualPath)
return nil, errors.Errorf("no mount path for an storage is: %s", mountPath)
}
return storageDriver, nil
}
@ -58,6 +58,7 @@ func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) {
}
// already has an id
err = initStorage(ctx, storage, storageDriver)
go callStorageHooks("add", storageDriver)
if err != nil {
return storage.ID, errors.Wrap(err, "failed init storage but storage is already created")
}
@ -77,6 +78,7 @@ func LoadStorage(ctx context.Context, storage model.Storage) error {
storageDriver := driverNew()
err = initStorage(ctx, storage, storageDriver)
go callStorageHooks("add", storageDriver)
log.Debugf("storage %+v is created", storageDriver)
return err
}
@ -145,6 +147,7 @@ func DisableStorage(ctx context.Context, id uint) error {
return errors.WithMessage(err, "failed update storage in db")
}
storagesMap.Delete(storage.MountPath)
go callStorageHooks("del", storageDriver)
return nil
}
@ -182,6 +185,7 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error {
}
err = initStorage(ctx, storage, storageDriver)
go callStorageHooks("update", storageDriver)
log.Debugf("storage %+v is update", storageDriver)
return err
}
@ -200,13 +204,14 @@ func DeleteStorageById(ctx context.Context, id uint) error {
if err := storageDriver.Drop(ctx); err != nil {
return errors.Wrapf(err, "failed drop storage")
}
// delete the storage in the memory
storagesMap.Delete(storage.MountPath)
go callStorageHooks("del", storageDriver)
}
// delete the storage in the database
if err := db.DeleteStorageById(id); err != nil {
return errors.WithMessage(err, "failed delete storage in database")
}
// delete the storage in the memory
storagesMap.Delete(storage.MountPath)
return nil
}

22
internal/search/build.go
View File

@ -8,9 +8,11 @@ import (
"sync/atomic"
"time"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/search/searcher"
"github.com/alist-org/alist/v3/pkg/mq"
"github.com/alist-org/alist/v3/pkg/utils"
mapset "github.com/deckarep/golang-set/v2"
@ -157,20 +159,15 @@ func Clear(ctx context.Context) error {
return instance.Clear(ctx)
}
func Config(ctx context.Context) searcher.Config {
return instance.Config()
}
func Update(parent string, objs []model.Obj) {
if instance == nil || !instance.Config().AutoUpdate || Running.Load() {
return
}
indexPaths := GetIndexPaths()
if !isIndexPath(parent, indexPaths) {
return
}
ignorePaths, err := GetIgnorePaths()
if err != nil {
log.Errorf("update search index error while get ignore paths: %+v", err)
return
}
if isIgnorePath(parent, ignorePaths) {
if isIgnorePath(parent) {
return
}
ctx := context.Background()
@ -219,7 +216,10 @@ func Update(parent string, objs []model.Obj) {
}
// build index if it's a folder
if objs[i].IsDir() {
err = BuildIndex(ctx, []string{path.Join(parent, objs[i].GetName())}, ignorePaths, -1, false)
err = BuildIndex(ctx,
[]string{path.Join(parent, objs[i].GetName())},
conf.SlicesMap[conf.IgnorePaths],
-1, false)
if err != nil {
log.Errorf("update search index error while build index: %+v", err)
return

40
internal/search/util.go
View File

@ -6,6 +6,7 @@ import (
"github.com/alist-org/alist/v3/drivers/alist_v3"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/setting"
@ -37,25 +38,7 @@ func WriteProgress(progress *model.IndexProgress) {
}
}
func GetIndexPaths() []string {
indexPaths := make([]string, 0)
customIndexPaths := setting.GetStr(conf.IndexPaths)
if customIndexPaths != "" {
indexPaths = append(indexPaths, strings.Split(customIndexPaths, "\n")...)
}
return indexPaths
}
func isIndexPath(path string, indexPaths []string) bool {
for _, indexPaths := range indexPaths {
if strings.HasPrefix(path, indexPaths) {
return true
}
}
return false
}
func GetIgnorePaths() ([]string, error) {
func updateIgnorePaths() {
storages := op.GetAllStorages()
ignorePaths := make([]string, 0)
var skipDrivers = []string{"AList V2", "AList V3", "Virtual"}
@ -84,14 +67,27 @@ func GetIgnorePaths() ([]string, error) {
if customIgnorePaths != "" {
ignorePaths = append(ignorePaths, strings.Split(customIgnorePaths, "\n")...)
}
return ignorePaths, nil
conf.SlicesMap[conf.IgnorePaths] = ignorePaths
}
func isIgnorePath(path string, ignorePaths []string) bool {
for _, ignorePath := range ignorePaths {
func isIgnorePath(path string) bool {
for _, ignorePath := range conf.SlicesMap[conf.IgnorePaths] {
if strings.HasPrefix(path, ignorePath) {
return true
}
}
return false
}
func init() {
op.RegisterSettingItemHook(conf.IgnorePaths, func(item *model.SettingItem) error {
updateIgnorePaths()
return nil
})
op.RegisterStorageHook(func(typ string, storage driver.Driver) {
var skipDrivers = []string{"AList V2", "AList V3", "Virtual"}
if utils.SliceContains(skipDrivers, storage.Config().Name) {
updateIgnorePaths()
}
})
}