ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: multiple search indexes (#2514)

Browse Source 
* refactor: abstract search interface

* wip: ~

* fix cycle import

* objs update hook

* wip: ~

* Delete search/none

* auto update index while cache changed

* db searcher

TODO: bleve init issue

cannot open index, metadata missing

* fix size type

why float64??

* fix typo

* fix nil pointer using

* api adapt ui

* bleve: fix clear & change struct
This commit is contained in:
Noah Hsu
2022-11-28 13:45:25 +08:00
committed by GitHub
parent bb969d8dc6
commit ddcba93eea
43 changed files with 855 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
server/common/req.go
View File

@ -1,20 +0,0 @@
package common
type PageReq struct {
Page int `json:"page" form:"page"`
PerPage int `json:"per_page" form:"per_page"`
}
const MaxUint = ^uint(0)
const MinUint = 0
const MaxInt = int(MaxUint >> 1)
const MinInt = -MaxInt - 1
func (p *PageReq) Validate() {
if p.Page < 1 {
p.Page = 1
}
if p.PerPage < 1 {
p.PerPage = MaxInt
}
}

16
server/handles/fsread.go
View File

@ -19,7 +19,7 @@ import (
)
type ListReq struct {
common.PageReq
model.PageReq
Path string `json:"path" form:"path"`
Password string `json:"password" form:"password"`
Refresh bool `json:"refresh"`
@ -86,7 +86,7 @@ func FsList(c *gin.Context) {
provider = storage.GetStorage().Driver
}
common.SuccessResp(c, FsListResp{
Content: toObjResp(objs, req.Path, isEncrypt(meta, req.Path)),
Content: toObjsResp(objs, req.Path, isEncrypt(meta, req.Path)),
Total: int64(total),
Readme: getReadme(meta, req.Path),
Write: user.CanWrite() || common.CanWrite(meta, req.Path),
@ -165,7 +165,7 @@ func isEncrypt(meta *model.Meta, path string) bool {
return true
}
func pagination(objs []model.Obj, req *common.PageReq) (int, []model.Obj) {
func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) {
pageIndex, pageSize := req.Page, req.PerPage
total := len(objs)
start := (pageIndex - 1) * pageSize
@ -179,17 +179,13 @@ func pagination(objs []model.Obj, req *common.PageReq) (int, []model.Obj) {
return total, objs[start:end]
}
func toObjResp(objs []model.Obj, parent string, encrypt bool) []ObjResp {
func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp {
var resp []ObjResp
for _, obj := range objs {
thumb := ""
if t, ok := obj.(model.Thumb); ok {
thumb = t.Thumb()
}
tp := conf.FOLDER
if !obj.IsDir() {
tp = utils.GetFileType(obj.GetName())
}
resp = append(resp, ObjResp{
Name: utils.MappingName(obj.GetName(), conf.FilenameCharMap),
Size: obj.GetSize(),
@ -197,7 +193,7 @@ func toObjResp(objs []model.Obj, parent string, encrypt bool) []ObjResp {
Modified: obj.ModTime(),
Sign: common.Sign(obj, parent, encrypt),
Thumb: thumb,
Type: tp,
Type: utils.GetObjType(obj.GetName(), obj.IsDir()),
})
}
return resp
@ -299,7 +295,7 @@ func FsGet(c *gin.Context) {
RawURL: rawURL,
Readme: getReadme(meta, req.Path),
Provider: provider,
Related: toObjResp(related, parentPath, isEncrypt(parentMeta, parentPath)),
Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)),
})
}

58
server/handles/index.go
View File

@ -2,53 +2,49 @@ package handles
import (
"context"
"strconv"
"github.com/alist-org/alist/v3/internal/db"
"github.com/alist-org/alist/v3/internal/index"
"github.com/alist-org/alist/v3/internal/search"
"github.com/alist-org/alist/v3/server/common"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
)
type BuildIndexReq struct {
Paths []string `json:"paths"`
MaxDepth int `json:"max_depth"`
IgnorePaths []string `json:"ignore_paths"`
}
func BuildIndex(c *gin.Context) {
var req BuildIndexReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return
}
if search.Running {
common.ErrorStrResp(c, "index is running", 400)
return
}
go func() {
// TODO: consider run build index as non-admin
user, _ := db.GetAdmin()
ctx := context.WithValue(c.Request.Context(), "user", user)
maxDepth, err := strconv.Atoi(c.PostForm("max_depth"))
ctx := context.Background()
err := search.Clear(ctx)
if err != nil {
maxDepth = -1
log.Errorf("clear index error: %+v", err)
return
}
err = search.BuildIndex(context.Background(), req.Paths, req.IgnorePaths, req.MaxDepth, true)
if err != nil {
log.Errorf("build index error: %+v", err)
}
indexPaths := []string{"/"}
ignorePaths := c.PostFormArray("ignore_paths")
index.BuildIndex(ctx, indexPaths, ignorePaths, maxDepth)
}()
common.SuccessResp(c)
}
func GetProgress(c *gin.Context) {
progress := index.ReadProgress()
common.SuccessResp(c, progress)
}
func Search(c *gin.Context) {
results := []string{}
query, exists := c.GetQuery("query")
if !exists {
common.SuccessResp(c, results)
}
sizeStr, _ := c.GetQuery("size")
size, err := strconv.Atoi(sizeStr)
if err != nil {
size = 10
}
searchResults, err := index.Search(query, size)
progress, err := search.Progress(c)
if err != nil {
common.ErrorResp(c, err, 500)
return
}
for _, documentMatch := range searchResults.Hits {
results = append(results, documentMatch.Fields["Path"].(string))
}
common.SuccessResp(c, results)
common.SuccessResp(c, progress)
}

2
server/handles/meta.go
View File

@ -15,7 +15,7 @@ import (
)
func ListMetas(c *gin.Context) {
var req common.PageReq
var req model.PageReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return

42
server/handles/search.go Normal file
View File

@ -0,0 +1,42 @@
package handles
import (
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/search"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/alist-org/alist/v3/server/common"
"github.com/gin-gonic/gin"
)
type SearchResp struct {
model.SearchNode
Type int `json:"type"`
}
func Search(c *gin.Context) {
var req model.SearchReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return
}
if err := req.Validate(); err != nil {
common.ErrorResp(c, err, 400)
return
}
nodes, total, err := search.Search(c, req)
if err != nil {
common.ErrorResp(c, err, 500)
return
}
common.SuccessResp(c, common.PageResp{
Content: utils.MustSliceConvert(nodes, nodeToSearchResp),
Total: total,
})
}
func nodeToSearchResp(node model.SearchNode) SearchResp {
return SearchResp{
SearchNode: node,
Type: utils.GetObjType(node.Name, node.IsDir),
}
}

2
server/handles/storage.go
View File

@ -12,7 +12,7 @@ import (
)
func ListStorages(c *gin.Context) {
var req common.PageReq
var req model.PageReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return

2
server/handles/user.go
View File

@ -11,7 +11,7 @@ import (
)
func ListUsers(c *gin.Context) {
var req common.PageReq
var req model.PageReq
if err := c.ShouldBind(&req); err != nil {
common.ErrorResp(c, err, 400)
return

19
server/middlewares/search.go Normal file
View File

@ -0,0 +1,19 @@
package middlewares
import (
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/setting"
"github.com/alist-org/alist/v3/server/common"
"github.com/gin-gonic/gin"
)
func SearchIndex(c *gin.Context) {
mode := setting.GetStr(conf.SearchIndex)
if mode == "none" {
common.ErrorResp(c, errs.SearchNotAvailable, 500)
c.Abort()
} else {
c.Next()
}
}

6
server/router.go
View File

@ -109,13 +109,13 @@ func admin(g *gin.RouterGroup) {
ms.POST("/send", message.HttpInstance.SendHandle)
index := g.Group("/index")
index.POST("/build", handles.BuildIndex)
index.GET("/progress", handles.GetProgress)
index.GET("/search", handles.Search)
index.POST("/build", middlewares.SearchIndex, handles.BuildIndex)
index.GET("/progress", middlewares.SearchIndex, handles.GetProgress)
}
func _fs(g *gin.RouterGroup) {
g.Any("/list", handles.FsList)
g.Any("/search", middlewares.SearchIndex, handles.Search)
g.Any("/get", handles.FsGet)
g.Any("/other", handles.FsOther)
g.Any("/dirs", handles.FsDirs)