feat(archive): archive manage (#7817)

* feat(archive): archive management * fix(ftp-server): remove duplicate ReadAtSeeker realization * fix(archive): bad seeking of SeekableStream * fix(archive): split internal and driver extraction api * feat(archive): patch * fix(shutdown): clear decompress upload tasks * chore * feat(archive): support .iso format * chore
2025-01-18 23:28:12 +08:00
parent ab22cf8233
commit bb40e2e2cd
36 changed files with 2854 additions and 127 deletions
--- a/internal/op/archive.go
+++ b/internal/op/archive.go
@ -0,0 +1,424 @@
+package op
+
+import (
+	"context"
+	stderrors "errors"
+	"github.com/alist-org/alist/v3/internal/archive/tool"
+	"github.com/alist-org/alist/v3/internal/stream"
+	"io"
+	stdpath "path"
+	"strings"
+	"time"
+
+	"github.com/Xhofe/go-cache"
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/errs"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/pkg/singleflight"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
+
+var archiveMetaCache = cache.NewMemCache(cache.WithShards[*model.ArchiveMetaProvider](64))
+var archiveMetaG singleflight.Group[*model.ArchiveMetaProvider]
+
+func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) {
+	if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
+		return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
+	}
+	path = utils.FixAndCleanPath(path)
+	key := Key(storage, path)
+	if !args.Refresh {
+		if meta, ok := archiveMetaCache.Get(key); ok {
+			log.Debugf("use cache when get %s archive meta", path)
+			return meta, nil
+		}
+	}
+	fn := func() (*model.ArchiveMetaProvider, error) {
+		_, m, err := getArchiveMeta(ctx, storage, path, args)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get %s archive met: %+v", path, err)
+		}
+		if !storage.Config().NoCache {
+			archiveMetaCache.Set(key, m, cache.WithEx[*model.ArchiveMetaProvider](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
+		}
+		return m, nil
+	}
+	if storage.Config().OnlyLocal {
+		meta, err := fn()
+		return meta, err
+	}
+	meta, err, _ := archiveMetaG.Do(key, fn)
+	return meta, err
+}
+
+func getArchiveToolAndStream(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (model.Obj, tool.Tool, *stream.SeekableStream, error) {
+	l, obj, err := Link(ctx, storage, path, args)
+	if err != nil {
+		return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] link", path)
+	}
+	ext := stdpath.Ext(obj.GetName())
+	t, err := tool.GetArchiveTool(ext)
+	if err != nil {
+		return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] archive tool", ext)
+	}
+	ss, err := stream.NewSeekableStream(stream.FileStream{Ctx: ctx, Obj: obj}, l)
+	if err != nil {
+		return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] stream", path)
+	}
+	return obj, t, ss, nil
+}
+
+func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (model.Obj, *model.ArchiveMetaProvider, error) {
+	storageAr, ok := storage.(driver.ArchiveReader)
+	if ok {
+		obj, err := GetUnwrap(ctx, storage, path)
+		if err != nil {
+			return nil, nil, errors.WithMessage(err, "failed to get file")
+		}
+		if obj.IsDir() {
+			return nil, nil, errors.WithStack(errs.NotFile)
+		}
+		meta, err := storageAr.GetArchiveMeta(ctx, obj, args.ArchiveArgs)
+		if !errors.Is(err, errs.NotImplement) {
+			return obj, &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: true}, err
+		}
+	}
+	obj, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer func() {
+		if err := ss.Close(); err != nil {
+			log.Errorf("failed to close file streamer, %v", err)
+		}
+	}()
+	meta, err := t.GetMeta(ss, args.ArchiveArgs)
+	return obj, &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: false}, err
+}
+
+var archiveListCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64))
+var archiveListG singleflight.Group[[]model.Obj]
+
+func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) {
+	if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
+		return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
+	}
+	path = utils.FixAndCleanPath(path)
+	metaKey := Key(storage, path)
+	key := stdpath.Join(metaKey, args.InnerPath)
+	if !args.Refresh {
+		if files, ok := archiveListCache.Get(key); ok {
+			log.Debugf("use cache when list archive [%s]%s", path, args.InnerPath)
+			return files, nil
+		}
+		if meta, ok := archiveMetaCache.Get(metaKey); ok {
+			log.Debugf("use meta cache when list archive [%s]%s", path, args.InnerPath)
+			return getChildrenFromArchiveMeta(meta, args.InnerPath)
+		}
+	}
+	objs, err, _ := archiveListG.Do(key, func() ([]model.Obj, error) {
+		obj, files, err := listArchive(ctx, storage, path, args)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to list archive [%s]%s: %+v", path, args.InnerPath, err)
+		}
+		// set path
+		for _, f := range files {
+			if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && obj.GetPath() != "" {
+				s.SetPath(stdpath.Join(obj.GetPath(), args.InnerPath, f.GetName()))
+			}
+		}
+		// warp obj name
+		model.WrapObjsName(files)
+		// sort objs
+		if storage.Config().LocalSort {
+			model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
+		}
+		model.ExtractFolder(files, storage.GetStorage().ExtractFolder)
+		if !storage.Config().NoCache {
+			if len(files) > 0 {
+				log.Debugf("set cache: %s => %+v", key, files)
+				archiveListCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
+			} else {
+				log.Debugf("del cache: %s", key)
+				archiveListCache.Del(key)
+			}
+		}
+		return files, nil
+	})
+	return objs, err
+}
+
+func _listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) {
+	storageAr, ok := storage.(driver.ArchiveReader)
+	if ok {
+		obj, err := GetUnwrap(ctx, storage, path)
+		if err != nil {
+			return nil, nil, errors.WithMessage(err, "failed to get file")
+		}
+		if obj.IsDir() {
+			return nil, nil, errors.WithStack(errs.NotFile)
+		}
+		files, err := storageAr.ListArchive(ctx, obj, args.ArchiveInnerArgs)
+		if !errors.Is(err, errs.NotImplement) {
+			return obj, files, err
+		}
+	}
+	obj, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer func() {
+		if err := ss.Close(); err != nil {
+			log.Errorf("failed to close file streamer, %v", err)
+		}
+	}()
+	files, err := t.List(ss, args.ArchiveInnerArgs)
+	return obj, files, err
+}
+
+func listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) {
+	obj, files, err := _listArchive(ctx, storage, path, args)
+	if errors.Is(err, errs.NotSupport) {
+		var meta model.ArchiveMeta
+		meta, err = GetArchiveMeta(ctx, storage, path, model.ArchiveMetaArgs{
+			ArchiveArgs: args.ArchiveArgs,
+			Refresh:     args.Refresh,
+		})
+		if err != nil {
+			return nil, nil, err
+		}
+		files, err = getChildrenFromArchiveMeta(meta, args.InnerPath)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	if err == nil && obj == nil {
+		obj, err = GetUnwrap(ctx, storage, path)
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+	return obj, files, err
+}
+
+func getChildrenFromArchiveMeta(meta model.ArchiveMeta, innerPath string) ([]model.Obj, error) {
+	obj := meta.GetTree()
+	if obj == nil {
+		return nil, errors.WithStack(errs.NotImplement)
+	}
+	dirs := splitPath(innerPath)
+	for _, dir := range dirs {
+		var next model.ObjTree
+		for _, c := range obj {
+			if c.GetName() == dir {
+				next = c
+				break
+			}
+		}
+		if next == nil {
+			return nil, errors.WithStack(errs.ObjectNotFound)
+		}
+		if !next.IsDir() || next.GetChildren() == nil {
+			return nil, errors.WithStack(errs.NotFolder)
+		}
+		obj = next.GetChildren()
+	}
+	return utils.SliceConvert(obj, func(src model.ObjTree) (model.Obj, error) {
+		return src, nil
+	})
+}
+
+func splitPath(path string) []string {
+	var parts []string
+	for {
+		dir, file := stdpath.Split(path)
+		if file == "" {
+			break
+		}
+		parts = append([]string{file}, parts...)
+		path = strings.TrimSuffix(dir, "/")
+	}
+	return parts
+}
+
+func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, model.Obj, error) {
+	if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
+		return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
+	}
+	path = utils.FixAndCleanPath(path)
+	af, err := GetUnwrap(ctx, storage, path)
+	if err != nil {
+		return nil, nil, errors.WithMessage(err, "failed to get file")
+	}
+	if af.IsDir() {
+		return nil, nil, errors.WithStack(errs.NotFile)
+	}
+	if g, ok := storage.(driver.ArchiveGetter); ok {
+		obj, err := g.ArchiveGet(ctx, af, args.ArchiveInnerArgs)
+		if err == nil {
+			return af, model.WrapObjName(obj), nil
+		}
+	}
+
+	if utils.PathEqual(args.InnerPath, "/") {
+		return af, &model.ObjWrapName{
+			Name: RootName,
+			Obj: &model.Object{
+				Name:     af.GetName(),
+				Path:     af.GetPath(),
+				ID:       af.GetID(),
+				Size:     af.GetSize(),
+				Modified: af.ModTime(),
+				IsFolder: true,
+			},
+		}, nil
+	}
+
+	innerDir, name := stdpath.Split(args.InnerPath)
+	args.InnerPath = strings.TrimSuffix(innerDir, "/")
+	files, err := ListArchive(ctx, storage, path, args)
+	if err != nil {
+		return nil, nil, errors.WithMessage(err, "failed get parent list")
+	}
+	for _, f := range files {
+		if f.GetName() == name {
+			return af, f, nil
+		}
+	}
+	return nil, nil, errors.WithStack(errs.ObjectNotFound)
+}
+
+type extractLink struct {
+	Link *model.Link
+	Obj  model.Obj
+}
+
+var extractCache = cache.NewMemCache(cache.WithShards[*extractLink](16))
+var extractG singleflight.Group[*extractLink]
+
+func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) {
+	if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
+		return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)
+	}
+	key := stdpath.Join(Key(storage, path), args.InnerPath)
+	if link, ok := extractCache.Get(key); ok {
+		return link.Link, link.Obj, nil
+	} else if link, ok := extractCache.Get(key + ":" + args.IP); ok {
+		return link.Link, link.Obj, nil
+	}
+	fn := func() (*extractLink, error) {
+		link, err := driverExtract(ctx, storage, path, args)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed extract archive")
+		}
+		if link.Link.Expiration != nil {
+			if link.Link.IPCacheKey {
+				key = key + ":" + args.IP
+			}
+			extractCache.Set(key, link, cache.WithEx[*extractLink](*link.Link.Expiration))
+		}
+		return link, nil
+	}
+	if storage.Config().OnlyLocal {
+		link, err := fn()
+		if err != nil {
+			return nil, nil, err
+		}
+		return link.Link, link.Obj, nil
+	}
+	link, err, _ := extractG.Do(key, fn)
+	if err != nil {
+		return nil, nil, err
+	}
+	return link.Link, link.Obj, err
+}
+
+func driverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*extractLink, error) {
+	storageAr, ok := storage.(driver.ArchiveReader)
+	if !ok {
+		return nil, errs.DriverExtractNotSupported
+	}
+	archiveFile, extracted, err := ArchiveGet(ctx, storage, path, model.ArchiveListArgs{
+		ArchiveInnerArgs: args,
+		Refresh:          false,
+	})
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to get file")
+	}
+	if extracted.IsDir() {
+		return nil, errors.WithStack(errs.NotFile)
+	}
+	link, err := storageAr.Extract(ctx, archiveFile, args)
+	return &extractLink{Link: link, Obj: extracted}, err
+}
+
+type streamWithParent struct {
+	rc     io.ReadCloser
+	parent *stream.SeekableStream
+}
+
+func (s *streamWithParent) Read(p []byte) (int, error) {
+	return s.rc.Read(p)
+}
+
+func (s *streamWithParent) Close() error {
+	err1 := s.rc.Close()
+	err2 := s.parent.Close()
+	return stderrors.Join(err1, err2)
+}
+
+func InternalExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) {
+	_, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs)
+	if err != nil {
+		return nil, 0, err
+	}
+	rc, size, err := t.Extract(ss, args)
+	if err != nil {
+		if e := ss.Close(); e != nil {
+			log.Errorf("failed to close file streamer, %v", e)
+		}
+		return nil, 0, err
+	}
+	return &streamWithParent{rc: rc, parent: ss}, size, nil
+}
+
+func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) error {
+	if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
+		return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
+	}
+	srcPath = utils.FixAndCleanPath(srcPath)
+	dstDirPath = utils.FixAndCleanPath(dstDirPath)
+	srcObj, err := GetUnwrap(ctx, storage, srcPath)
+	if err != nil {
+		return errors.WithMessage(err, "failed to get src object")
+	}
+	dstDir, err := GetUnwrap(ctx, storage, dstDirPath)
+	if err != nil {
+		return errors.WithMessage(err, "failed to get dst dir")
+	}
+
+	switch s := storage.(type) {
+	case driver.ArchiveDecompressResult:
+		var newObjs []model.Obj
+		newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args)
+		if err == nil {
+			if newObjs != nil && len(newObjs) > 0 {
+				for _, newObj := range newObjs {
+					addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))
+				}
+			} else if !utils.IsBool(lazyCache...) {
+				ClearCache(storage, dstDirPath)
+			}
+		}
+	case driver.ArchiveDecompress:
+		err = s.ArchiveDecompress(ctx, srcObj, dstDir, args)
+		if err == nil && !utils.IsBool(lazyCache...) {
+			ClearCache(storage, dstDirPath)
+		}
+	default:
+		return errs.NotImplement
+	}
+	return errors.WithStack(err)
+}