ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: optimize index build

Browse Source
This commit is contained in:
Noah Hsu
2022-12-05 15:46:34 +08:00
parent bc6baf1be0
commit bd33c200dc
6 changed files with 215 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
internal/search/build.go
View File

@ -4,16 +4,20 @@ import (
"context"
"path"
"path/filepath"
"strings"
"sync/atomic"
"time"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/mq"
"github.com/alist-org/alist/v3/pkg/utils"
log "github.com/sirupsen/logrus"
)
var (
Running = false
Running = atomic.Bool{}
Quit chan struct{}
)
@ -22,8 +26,9 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
if err != nil {
return err
}
var skipDrivers = []string{"AList V2", "AList V3"}
for _, storage := range storages {
if storage.Driver == "AList V2" || storage.Driver == "AList V3" {
if utils.SliceContains(skipDrivers, storage.Driver) {
// TODO: request for indexing permission
ignorePaths = append(ignorePaths, storage.MountPath)
}
@ -32,66 +37,72 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
objCount uint64 = 0
fi model.Obj
)
Running = true
Running.Store(true)
Quit = make(chan struct{}, 1)
parents := []string{}
infos := []model.Obj{}
indexMQ := mq.NewInMemoryMQ[ObjWithParent]()
go func() {
ticker := time.NewTicker(5 * time.Second)
for {
select {
case <-ticker.C:
log.Infof("index obj count: %d", objCount)
if len(parents) != 0 {
log.Debugf("current index: %s", parents[len(parents)-1])
}
if err = BatchIndex(ctx, parents, infos); err != nil {
log.Errorf("build index in batch error: %+v", err)
} else {
objCount = objCount + uint64(len(parents))
}
if count {
WriteProgress(&model.IndexProgress{
ObjCount: objCount,
IsDone: false,
LastDoneTime: nil,
})
}
parents = nil
infos = nil
indexMQ.ConsumeAll(func(messages []mq.Message[ObjWithParent]) {
if len(messages) != 0 {
log.Debugf("current index: %s", messages[len(messages)-1].Content.Parent)
}
if err = BatchIndex(ctx, utils.MustSliceConvert(messages,
func(src mq.Message[ObjWithParent]) ObjWithParent {
return src.Content
})); err != nil {
log.Errorf("build index in batch error: %+v", err)
} else {
objCount = objCount + uint64(len(messages))
}
if count {
WriteProgress(&model.IndexProgress{
ObjCount: objCount,
IsDone: false,
LastDoneTime: nil,
})
}
})
case <-Quit:
Running = false
Running.Store(false)
ticker.Stop()
eMsg := ""
now := time.Now()
originErr := err
if err = BatchIndex(ctx, parents, infos); err != nil {
log.Errorf("build index in batch error: %+v", err)
} else {
objCount = objCount + uint64(len(parents))
}
parents = nil
infos = nil
if originErr != nil {
log.Errorf("build index error: %+v", err)
eMsg = err.Error()
} else {
log.Infof("success build index, count: %d", objCount)
}
if count {
WriteProgress(&model.IndexProgress{
ObjCount: objCount,
IsDone: originErr == nil,
LastDoneTime: &now,
Error: eMsg,
})
}
indexMQ.ConsumeAll(func(messages []mq.Message[ObjWithParent]) {
if err = BatchIndex(ctx, utils.MustSliceConvert(messages,
func(src mq.Message[ObjWithParent]) ObjWithParent {
return src.Content
})); err != nil {
log.Errorf("build index in batch error: %+v", err)
} else {
objCount = objCount + uint64(len(messages))
}
if originErr != nil {
log.Errorf("build index error: %+v", err)
eMsg = err.Error()
} else {
log.Infof("success build index, count: %d", objCount)
}
if count {
WriteProgress(&model.IndexProgress{
ObjCount: objCount,
IsDone: true,
LastDoneTime: &now,
Error: eMsg,
})
}
})
return
}
}
}()
defer func() {
if Running {
if Running.Load() {
Quit <- struct{}{}
}
}()
@ -107,8 +118,11 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
}
for _, indexPath := range indexPaths {
walkFn := func(indexPath string, info model.Obj) error {
if !Running.Load() {
return filepath.SkipDir
}
for _, avoidPath := range ignorePaths {
if indexPath == avoidPath {
if strings.HasPrefix(indexPath, avoidPath) {
return filepath.SkipDir
}
}
@ -116,8 +130,12 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
if indexPath == "/" {
return nil
}
parents = append(parents, path.Dir(indexPath))
infos = append(infos, info)
indexMQ.Publish(mq.Message[ObjWithParent]{
Content: ObjWithParent{
Obj: info,
Parent: path.Dir(indexPath),
},
})
return nil
}
fi, err = fs.Get(ctx, indexPath)