diff --git a/.github/workflows/auto_lang.yml b/.github/workflows/auto_lang.yml index 24f6ba06..439ebb77 100644 --- a/.github/workflows/auto_lang.yml +++ b/.github/workflows/auto_lang.yml @@ -7,6 +7,7 @@ on: paths: - 'drivers/**' - 'internal/bootstrap/data/setting.go' + - 'internal/conf/const.go' workflow_dispatch: jobs: @@ -69,4 +70,4 @@ jobs: github_token: ${{ secrets.MY_TOKEN }} branch: main directory: alist-web - repository: alist-org/alist-web \ No newline at end of file + repository: alist-org/alist-web diff --git a/internal/bootstrap/index.go b/internal/bootstrap/index.go index 02a73318..02796c72 100644 --- a/internal/bootstrap/index.go +++ b/internal/bootstrap/index.go @@ -1,5 +1,18 @@ package bootstrap +import ( + "github.com/alist-org/alist/v3/internal/search" + log "github.com/sirupsen/logrus" +) + func InitIndex() { - // TODO init ? Probably not. + progress, err := search.Progress() + if err != nil { + log.Errorf("init index error: %+v", err) + return + } + if !progress.IsDone { + progress.IsDone = true + search.WriteProgress(progress) + } } diff --git a/internal/conf/const.go b/internal/conf/const.go index dab7b9d9..495ac2bd 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -25,7 +25,7 @@ const ( AudioTypes = "audio_types" VideoTypes = "video_types" ImageTypes = "image_types" - //OfficeTypes = "office_types" + // OfficeTypes = "office_types" ProxyTypes = "proxy_types" OfficeViewers = "office_viewers" PdfViewers = "pdf_viewers" diff --git a/internal/db/searchnode.go b/internal/db/searchnode.go index a37fce9c..2345dadf 100644 --- a/internal/db/searchnode.go +++ b/internal/db/searchnode.go @@ -11,6 +11,10 @@ func CreateSearchNode(node *model.SearchNode) error { return db.Create(node).Error } +func BatchCreateSearchNodes(nodes *[]model.SearchNode) error { + return db.CreateInBatches(nodes, 1000).Error +} + func DeleteSearchNodesByParent(parent string) error { return db.Where(fmt.Sprintf("%s LIKE ?", columnName("path")), fmt.Sprintf("%s%%", parent)). diff --git a/internal/fs/walk.go b/internal/fs/walk.go index 1404c34a..53d1aec0 100644 --- a/internal/fs/walk.go +++ b/internal/fs/walk.go @@ -14,9 +14,9 @@ import ( // WalkFS will stop when current depth > `depth`. For each visited node, // WalkFS calls walkFn. If a visited file system node is a directory and // walkFn returns path.SkipDir, walkFS will skip traversal of this node. -func WalkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj, err error) error) error { +func WalkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn func(reqPath string, info model.Obj) error) error { // This implementation is based on Walk's code in the standard path/path package. - walkFnErr := walkFn(name, info, nil) + walkFnErr := walkFn(name, info) if walkFnErr != nil { if info.IsDir() && walkFnErr == filepath.SkipDir { return nil diff --git a/internal/model/search.go b/internal/model/search.go index dd467251..409fb3ee 100644 --- a/internal/model/search.go +++ b/internal/model/search.go @@ -34,3 +34,7 @@ func (p *SearchReq) Validate() error { } return nil } + +func (s *SearchNode) Type() string { + return "SearchNode" +} diff --git a/internal/search/bleve/init.go b/internal/search/bleve/init.go index 07203685..e764c3cf 100644 --- a/internal/search/bleve/init.go +++ b/internal/search/bleve/init.go @@ -17,6 +17,15 @@ func Init(indexPath *string) (bleve.Index, error) { if err == bleve.ErrorIndexPathDoesNotExist { log.Infof("Creating new index...") indexMapping := bleve.NewIndexMapping() + searchNodeMapping := bleve.NewDocumentMapping() + searchNodeMapping.AddFieldMappingsAt("is_dir", bleve.NewBooleanFieldMapping()) + // TODO: appoint analyzer + parentFieldMapping := bleve.NewTextFieldMapping() + searchNodeMapping.AddFieldMappingsAt("parent", parentFieldMapping) + // TODO: appoint analyzer + nameFieldMapping := bleve.NewKeywordFieldMapping() + searchNodeMapping.AddFieldMappingsAt("name", nameFieldMapping) + indexMapping.AddDocumentMapping("SearchNode", searchNodeMapping) fileIndex, err = bleve.New(*indexPath, indexMapping) if err != nil { return nil, err diff --git a/internal/search/bleve/search.go b/internal/search/bleve/search.go index 083a644f..c0f173f8 100644 --- a/internal/search/bleve/search.go +++ b/internal/search/bleve/search.go @@ -49,6 +49,14 @@ func (b *Bleve) Index(ctx context.Context, node model.SearchNode) error { return b.BIndex.Index(uuid.NewString(), node) } +func (b *Bleve) BatchIndex(ctx context.Context, nodes []model.SearchNode) error { + batch := b.BIndex.NewBatch() + for _, node := range nodes { + batch.Index(uuid.NewString(), node) + } + return b.BIndex.Batch(batch) +} + func (b *Bleve) Get(ctx context.Context, parent string) ([]model.SearchNode, error) { return nil, errs.NotSupport } diff --git a/internal/search/build.go b/internal/search/build.go index aba92865..922438b8 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -14,32 +14,85 @@ import ( var ( Running = false + Quit chan struct{} ) func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int, count bool) error { - var objCount uint64 = 0 - Running = true - var ( - err error - fi model.Obj - ) - defer func() { - Running = false - now := time.Now() - eMsg := "" - if err != nil { - log.Errorf("build index error: %+v", err) - eMsg = err.Error() - } else { - log.Infof("success build index, count: %d", objCount) + storages, err := db.GetEnabledStorages() + if err != nil { + return err + } + for _, storage := range storages { + if storage.Driver == "AList V2" || storage.Driver == "AList V3" { + // TODO: request for indexing permission + ignorePaths = append(ignorePaths, storage.MountPath) } - if count { - WriteProgress(&model.IndexProgress{ - ObjCount: objCount, - IsDone: err == nil, - LastDoneTime: &now, - Error: eMsg, - }) + } + var ( + objCount uint64 = 0 + fi model.Obj + ) + Running = true + Quit = make(chan struct{}, 1) + parents := []string{} + infos := []model.Obj{} + go func() { + ticker := time.NewTicker(5 * time.Second) + for { + select { + case <-ticker.C: + log.Infof("index obj count: %d", objCount) + if len(parents) != 0 { + log.Debugf("current index: %s", parents[len(parents)-1]) + } + if err = BatchIndex(ctx, parents, infos); err != nil { + log.Errorf("build index in batch error: %+v", err) + } else { + objCount = objCount + uint64(len(parents)) + } + if count { + WriteProgress(&model.IndexProgress{ + ObjCount: objCount, + IsDone: false, + LastDoneTime: nil, + }) + } + parents = nil + infos = nil + case <-Quit: + Running = false + ticker.Stop() + eMsg := "" + now := time.Now() + originErr := err + if err = BatchIndex(ctx, parents, infos); err != nil { + log.Errorf("build index in batch error: %+v", err) + } else { + objCount = objCount + uint64(len(parents)) + } + parents = nil + infos = nil + if originErr != nil { + log.Errorf("build index error: %+v", err) + eMsg = err.Error() + } else { + log.Infof("success build index, count: %d", objCount) + } + if count { + WriteProgress(&model.IndexProgress{ + ObjCount: objCount, + IsDone: originErr == nil, + LastDoneTime: &now, + Error: eMsg, + }) + } + return + } + } + }() + defer func() { + if Running { + Quit <- struct{}{} } }() admin, err := db.GetAdmin() @@ -53,7 +106,7 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth }) } for _, indexPath := range indexPaths { - walkFn := func(indexPath string, info model.Obj, err error) error { + walkFn := func(indexPath string, info model.Obj) error { for _, avoidPath := range ignorePaths { if indexPath == avoidPath { return filepath.SkipDir @@ -63,23 +116,8 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth if indexPath == "/" { return nil } - err = Index(ctx, path.Dir(indexPath), info) - if err != nil { - return err - } else { - objCount++ - } - if objCount%100 == 0 { - log.Infof("index obj count: %d", objCount) - log.Debugf("current success index: %s", indexPath) - if count { - WriteProgress(&model.IndexProgress{ - ObjCount: objCount, - IsDone: false, - LastDoneTime: nil, - }) - } - } + parents = append(parents, path.Dir(indexPath)) + infos = append(infos, info) return nil } fi, err = fs.Get(ctx, indexPath) diff --git a/internal/search/db/search.go b/internal/search/db/search.go index a09ee6ca..f3de0750 100644 --- a/internal/search/db/search.go +++ b/internal/search/db/search.go @@ -22,6 +22,10 @@ func (D DB) Index(ctx context.Context, node model.SearchNode) error { return db.CreateSearchNode(&node) } +func (D DB) BatchIndex(ctx context.Context, nodes []model.SearchNode) error { + return db.BatchCreateSearchNodes(&nodes) +} + func (D DB) Get(ctx context.Context, parent string) ([]model.SearchNode, error) { return db.GetSearchNodesByParent(parent) } diff --git a/internal/search/progress.go b/internal/search/progress.go index 85cc4f37..65d1b8be 100644 --- a/internal/search/progress.go +++ b/internal/search/progress.go @@ -1,8 +1,6 @@ package search import ( - "context" - "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" @@ -11,7 +9,7 @@ import ( log "github.com/sirupsen/logrus" ) -func Progress(ctx context.Context) (*model.IndexProgress, error) { +func Progress() (*model.IndexProgress, error) { p := setting.GetStr(conf.IndexProgress) var progress model.IndexProgress err := utils.Json.UnmarshalFromString(p, &progress) diff --git a/internal/search/search.go b/internal/search/search.go index 6f522967..9b84cf2b 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -59,6 +59,25 @@ func Index(ctx context.Context, parent string, obj model.Obj) error { }) } +func BatchIndex(ctx context.Context, parents []string, objs []model.Obj) error { + if instance == nil { + return errs.SearchNotAvailable + } + if len(parents) == 0 { + return nil + } + searchNodes := []model.SearchNode{} + for i := range parents { + searchNodes = append(searchNodes, model.SearchNode{ + Parent: parents[i], + Name: objs[i].GetName(), + IsDir: objs[i].IsDir(), + Size: objs[i].GetSize(), + }) + } + return instance.BatchIndex(ctx, searchNodes) +} + func init() { db.RegisterSettingItemHook(conf.SearchIndex, func(item *model.SettingItem) error { log.Debugf("searcher init, mode: %s", item.Value) diff --git a/internal/search/searcher/searcher.go b/internal/search/searcher/searcher.go index 96fa6a36..a6c34a0a 100644 --- a/internal/search/searcher/searcher.go +++ b/internal/search/searcher/searcher.go @@ -18,6 +18,8 @@ type Searcher interface { Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) // Index obj with parent Index(ctx context.Context, node model.SearchNode) error + // Index obj with parent in batches + BatchIndex(ctx context.Context, nodes []model.SearchNode) error // Get by parent Get(ctx context.Context, parent string) ([]model.SearchNode, error) // Del with prefix diff --git a/internal/search/update.go b/internal/search/update.go index bec8e347..e512a9f9 100644 --- a/internal/search/update.go +++ b/internal/search/update.go @@ -16,7 +16,7 @@ func Update(parent string, objs []model.Obj) { } ctx := context.Background() // only update when index have built - progress, err := Progress(ctx) + progress, err := Progress() if err != nil { log.Errorf("update search index error while get progress: %+v", err) return diff --git a/server/handles/index.go b/server/handles/index.go index 1c924b31..46f7b3aa 100644 --- a/server/handles/index.go +++ b/server/handles/index.go @@ -40,8 +40,17 @@ func BuildIndex(c *gin.Context) { common.SuccessResp(c) } +func StopIndex(c *gin.Context) { + if !search.Running { + common.ErrorStrResp(c, "index is not running", 400) + return + } + search.Quit <- struct{}{} + common.SuccessResp(c) +} + func GetProgress(c *gin.Context) { - progress, err := search.Progress(c) + progress, err := search.Progress() if err != nil { common.ErrorResp(c, err, 500) return diff --git a/server/router.go b/server/router.go index 4158387b..7b90ba44 100644 --- a/server/router.go +++ b/server/router.go @@ -110,6 +110,7 @@ func admin(g *gin.RouterGroup) { index := g.Group("/index") index.POST("/build", middlewares.SearchIndex, handles.BuildIndex) + index.POST("/stop", middlewares.SearchIndex, handles.StopIndex) index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) }