feat(search): search with meilisearch (#6060)

* feat(search): search with meilisearch.

* feat(search): meilisearch supports auto update.

* chores: remove utils.Log.

* fix(search): the null pointer caused by deleting non-existing file/folder indexes.

---------

Co-authored-by: Andy Hsu <i@nn.ci>

internal/bootstrap/data/setting.go

@@ -145,7 +145,7 @@ func InitialSettings() []model.SettingItem {
 
 		// single settings
 		{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
-		{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,none", Group: model.INDEX},
+		{Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,meilisearch,none", Group: model.INDEX},
 		{Key: conf.AutoUpdateIndex, Value: "false", Type: conf.TypeBool, Group: model.INDEX},
 		{Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`},
 		{Key: conf.MaxIndexDepth, Value: "20", Type: conf.TypeNumber, Group: model.INDEX, Flag: model.PRIVATE, Help: `max depth of index`},

internal/conf/config.go

@@ -1,10 +1,9 @@
 package conf
 
 import (
-	"path/filepath"
-
 	"github.com/alist-org/alist/v3/cmd/flags"
 	"github.com/alist-org/alist/v3/pkg/utils/random"
+	"path/filepath"
 )
 
 type Database struct {
@@ -20,6 +19,12 @@ type Database struct {
 	DSN         string `json:"dsn" env:"DSN"`
 }
 
+type Meilisearch struct {
+	Host        string `json:"host" env:"HOST"`
+	APIKey      string `json:"api_key" env:"API_KEY"`
+	IndexPrefix string `json:"index_prefix" env:"INDEX_PREFIX"`
+}
+
 type Scheme struct {
 	Address      string `json:"address" env:"ADDR"`
 	HttpPort     int    `json:"http_port" env:"HTTP_PORT"`
@@ -65,6 +70,7 @@ type Config struct {
 	JwtSecret             string      `json:"jwt_secret" env:"JWT_SECRET"`
 	TokenExpiresIn        int         `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"`
 	Database              Database    `json:"database" envPrefix:"DB_"`
+	Meilisearch           Meilisearch `json:"meilisearch" env:"MEILISEARCH"`
 	Scheme                Scheme      `json:"scheme"`
 	TempDir               string      `json:"temp_dir" env:"TEMP_DIR"`
 	BleveDir              string      `json:"bleve_dir" env:"BLEVE_DIR"`
@@ -101,6 +107,9 @@ func DefaultConfig() *Config {
 			TablePrefix: "x_",
 			DBFile:      dbPath,
 		},
+		Meilisearch: Meilisearch{
+			Host: "http://localhost:7700",
+		},
 		BleveDir: indexDir,
 		Log: LogConfig{
 			Enable:     true,

internal/search/import.go

@@ -4,4 +4,5 @@ import (
 	_ "github.com/alist-org/alist/v3/internal/search/bleve"
 	_ "github.com/alist-org/alist/v3/internal/search/db"
 	_ "github.com/alist-org/alist/v3/internal/search/db_non_full_text"
+	_ "github.com/alist-org/alist/v3/internal/search/meilisearch"
 )

internal/search/meilisearch/init.go

@@ -0,0 +1,89 @@
+package meilisearch
+
+import (
+	"errors"
+	"fmt"
+	"github.com/alist-org/alist/v3/internal/conf"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/search/searcher"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/meilisearch/meilisearch-go"
+)
+
+var config = searcher.Config{
+	Name:       "meilisearch",
+	AutoUpdate: true,
+}
+
+func init() {
+	searcher.RegisterSearcher(config, func() (searcher.Searcher, error) {
+		m := Meilisearch{
+			Client: meilisearch.NewClient(meilisearch.ClientConfig{
+				Host:   conf.Conf.Meilisearch.Host,
+				APIKey: conf.Conf.Meilisearch.APIKey,
+			}),
+			IndexUid:             conf.Conf.Meilisearch.IndexPrefix + "alist",
+			FilterableAttributes: []string{"parent", "is_dir", "name"},
+			SearchableAttributes: []string{"name"},
+		}
+
+		_, err := m.Client.GetIndex(m.IndexUid)
+		if err != nil {
+			var mErr *meilisearch.Error
+			ok := errors.As(err, &mErr)
+			if ok && mErr.MeilisearchApiError.Code == "index_not_found" {
+				task, err := m.Client.CreateIndex(&meilisearch.IndexConfig{
+					Uid:        m.IndexUid,
+					PrimaryKey: "id",
+				})
+				if err != nil {
+					return nil, err
+				}
+				forTask, err := m.Client.WaitForTask(task.TaskUID)
+				if err != nil {
+					return nil, err
+				}
+				if forTask.Status != meilisearch.TaskStatusSucceeded {
+					return nil, fmt.Errorf("index creation failed, task status is %s", forTask.Status)
+				}
+			} else {
+				return nil, err
+			}
+		}
+		attributes, err := m.Client.Index(m.IndexUid).GetFilterableAttributes()
+		if err != nil {
+			return nil, err
+		}
+		if attributes == nil || !utils.SliceAllContains(*attributes, m.FilterableAttributes...) {
+			_, err = m.Client.Index(m.IndexUid).UpdateFilterableAttributes(&m.FilterableAttributes)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		attributes, err = m.Client.Index(m.IndexUid).GetSearchableAttributes()
+		if err != nil {
+			return nil, err
+		}
+		if attributes == nil || !utils.SliceAllContains(*attributes, m.SearchableAttributes...) {
+			_, err = m.Client.Index(m.IndexUid).UpdateSearchableAttributes(&m.SearchableAttributes)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		pagination, err := m.Client.Index(m.IndexUid).GetPagination()
+		if err != nil {
+			return nil, err
+		}
+		if pagination.MaxTotalHits != int64(model.MaxInt) {
+			_, err := m.Client.Index(m.IndexUid).UpdatePagination(&meilisearch.Pagination{
+				MaxTotalHits: int64(model.MaxInt),
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+		return &m, nil
+	})
+}

internal/search/meilisearch/search.go

@@ -0,0 +1,227 @@
+package meilisearch
+
+import (
+	"context"
+	"fmt"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/internal/search/searcher"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/google/uuid"
+	"github.com/meilisearch/meilisearch-go"
+	"path"
+	"strings"
+	"time"
+)
+
+type searchDocument struct {
+	ID string `json:"id"`
+	model.SearchNode
+}
+
+type Meilisearch struct {
+	Client               *meilisearch.Client
+	IndexUid             string
+	FilterableAttributes []string
+	SearchableAttributes []string
+}
+
+func (m *Meilisearch) Config() searcher.Config {
+	return config
+}
+
+func (m *Meilisearch) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) {
+	mReq := &meilisearch.SearchRequest{
+		AttributesToSearchOn: m.SearchableAttributes,
+		Page:                 int64(req.Page),
+		HitsPerPage:          int64(req.PerPage),
+	}
+	if req.Scope != 0 {
+		mReq.Filter = fmt.Sprintf("is_dir = %v", req.Scope == 1)
+	}
+	search, err := m.Client.Index(m.IndexUid).Search(req.Keywords, mReq)
+	if err != nil {
+		return nil, 0, err
+	}
+	nodes, err := utils.SliceConvert(search.Hits, func(src any) (model.SearchNode, error) {
+		srcMap := src.(map[string]any)
+		return model.SearchNode{
+			Parent: srcMap["parent"].(string),
+			Name:   srcMap["name"].(string),
+			IsDir:  srcMap["is_dir"].(bool),
+			Size:   int64(srcMap["size"].(float64)),
+		}, nil
+	})
+	if err != nil {
+		return nil, 0, err
+	}
+	return nodes, search.TotalHits, nil
+}
+
+func (m *Meilisearch) Index(ctx context.Context, node model.SearchNode) error {
+	return m.BatchIndex(ctx, []model.SearchNode{node})
+}
+
+func (m *Meilisearch) BatchIndex(ctx context.Context, nodes []model.SearchNode) error {
+	documents, _ := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) {
+
+		return &searchDocument{
+			ID:         uuid.NewString(),
+			SearchNode: src,
+		}, nil
+	})
+
+	_, err := m.Client.Index(m.IndexUid).AddDocuments(documents)
+	if err != nil {
+		return err
+	}
+
+	//// Wait for the task to complete and check
+	//forTask, err := m.Client.WaitForTask(task.TaskUID, meilisearch.WaitParams{
+	//	Context:  ctx,
+	//	Interval: time.Millisecond * 50,
+	//})
+	//if err != nil {
+	//	return err
+	//}
+	//if forTask.Status != meilisearch.TaskStatusSucceeded {
+	//	return fmt.Errorf("BatchIndex failed, task status is %s", forTask.Status)
+	//}
+	return nil
+}
+
+func (m *Meilisearch) getDocumentsByParent(ctx context.Context, parent string) ([]*searchDocument, error) {
+	var result meilisearch.DocumentsResult
+	err := m.Client.Index(m.IndexUid).GetDocuments(&meilisearch.DocumentsQuery{
+		Filter: fmt.Sprintf("parent = '%s'", strings.ReplaceAll(parent, "'", "\\'")),
+		Limit:  int64(model.MaxInt),
+	}, &result)
+	if err != nil {
+		return nil, err
+	}
+	return utils.SliceConvert(result.Results, func(src map[string]any) (*searchDocument, error) {
+		return &searchDocument{
+			ID: src["id"].(string),
+			SearchNode: model.SearchNode{
+				Parent: src["parent"].(string),
+				Name:   src["name"].(string),
+				IsDir:  src["is_dir"].(bool),
+				Size:   int64(src["size"].(float64)),
+			},
+		}, nil
+	})
+}
+
+func (m *Meilisearch) Get(ctx context.Context, parent string) ([]model.SearchNode, error) {
+	result, err := m.getDocumentsByParent(ctx, parent)
+	if err != nil {
+		return nil, err
+	}
+	return utils.SliceConvert(result, func(src *searchDocument) (model.SearchNode, error) {
+		return src.SearchNode, nil
+	})
+
+}
+
+func (m *Meilisearch) getParentsByPrefix(ctx context.Context, parent string) ([]string, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+		parents := []string{parent}
+		get, err := m.getDocumentsByParent(ctx, parent)
+		if err != nil {
+			return nil, err
+		}
+		for _, node := range get {
+			if node.IsDir {
+				arr, err := m.getParentsByPrefix(ctx, path.Join(node.Parent, node.Name))
+				if err != nil {
+					return nil, err
+				}
+				parents = append(parents, arr...)
+			}
+		}
+		return parents, nil
+	}
+}
+
+func (m *Meilisearch) DelDirChild(ctx context.Context, prefix string) error {
+	dfs, err := m.getParentsByPrefix(ctx, utils.FixAndCleanPath(prefix))
+	if err != nil {
+		return err
+	}
+	utils.SliceReplace(dfs, func(src string) string {
+		return "'" + strings.ReplaceAll(src, "'", "\\'") + "'"
+	})
+	s := fmt.Sprintf("parent IN [%s]", strings.Join(dfs, ","))
+	task, err := m.Client.Index(m.IndexUid).DeleteDocumentsByFilter(s)
+	if err != nil {
+		return err
+	}
+	taskStatus, err := m.getTaskStatus(ctx, task.TaskUID)
+	if err != nil {
+		return err
+	}
+	if taskStatus != meilisearch.TaskStatusSucceeded {
+		return fmt.Errorf("DelDir failed, task status is %s", taskStatus)
+	}
+	return nil
+}
+
+func (m *Meilisearch) Del(ctx context.Context, prefix string) error {
+	prefix = utils.FixAndCleanPath(prefix)
+	dir, name := path.Split(prefix)
+	get, err := m.getDocumentsByParent(ctx, dir[:len(dir)-1])
+	if err != nil {
+		return err
+	}
+	var document *searchDocument
+	for _, v := range get {
+		if v.Name == name {
+			document = v
+			break
+		}
+	}
+	if document == nil {
+		// Defensive programming. Document may be the folder, try deleting Child
+		return m.DelDirChild(ctx, prefix)
+	}
+	if document.IsDir {
+		err = m.DelDirChild(ctx, prefix)
+		if err != nil {
+			return err
+		}
+	}
+	task, err := m.Client.Index(m.IndexUid).DeleteDocument(document.ID)
+	if err != nil {
+		return err
+	}
+	taskStatus, err := m.getTaskStatus(ctx, task.TaskUID)
+	if err != nil {
+		return err
+	}
+	if taskStatus != meilisearch.TaskStatusSucceeded {
+		return fmt.Errorf("DelDir failed, task status is %s", taskStatus)
+	}
+	return nil
+}
+
+func (m *Meilisearch) Release(ctx context.Context) error {
+	return nil
+}
+
+func (m *Meilisearch) Clear(ctx context.Context) error {
+	_, err := m.Client.Index(m.IndexUid).DeleteAllDocuments()
+	return err
+}
+
+func (m *Meilisearch) getTaskStatus(ctx context.Context, taskUID int64) (meilisearch.TaskStatus, error) {
+	forTask, err := m.Client.WaitForTask(taskUID, meilisearch.WaitParams{
+		Context:  ctx,
+		Interval: time.Second,
+	})
+	if err != nil {
+		return meilisearch.TaskStatusUnknown, err
+	}
+	return forTask.Status, nil
+}