feat: misc improvements about upload/copy/hash (#5045)

general: add createTime/updateTime support in webdav and some drivers
general: add hash support in some drivers
general: cross-storage rapid-upload support
general: enhance upload to avoid local temp file if possible
general: replace readseekcloser with File interface to speed upstream operations
feat(aliyun_open): same as above
feat(crypt): add hack for 139cloud

Close #4934 
Close #4819 

baidu_netdisk needs to improve the upload code to support rapid-upload

internal/model/args.go

@@ -1,6 +1,7 @@
 package model
 
 import (
+	"context"
 	"io"
 	"net/http"
 	"time"
@@ -22,13 +23,14 @@ type LinkArgs struct {
 }
 
 type Link struct {
-	URL             string            `json:"url"`
-	Header          http.Header       `json:"header"` // needed header (for url) or response header(for data or writer)
-	RangeReadCloser RangeReadCloser   `json:"-"`      // recommended way
-	ReadSeekCloser  io.ReadSeekCloser `json:"-"`      // best for local,smb... file system, which exposes ReadSeekCloser
+	URL             string            `json:"url"`    // most common way
+	Header          http.Header       `json:"header"` // needed header (for url)
+	RangeReadCloser RangeReadCloserIF `json:"-"`      // recommended way if can't use URL
+	MFile           File              `json:"-"`      // best for local,smb... file system, which exposes MFile
 
 	Expiration *time.Duration // local cache expire Duration
 	IPCacheKey bool           `json:"-"` // add ip to cache key
+
 	//for accelerating request, use multi-thread downloading
 	Concurrency int `json:"concurrency"`
 	PartSize    int `json:"part_size"`
@@ -45,10 +47,23 @@ type FsOtherArgs struct {
 	Method string      `json:"method" form:"method"`
 	Data   interface{} `json:"data" form:"data"`
 }
-type RangeReadCloser struct {
-	RangeReader RangeReaderFunc
-	Closers     *utils.Closers
+type RangeReadCloserIF interface {
+	RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error)
+	utils.ClosersIF
 }
 
-type WriterFunc func(w io.Writer) error
-type RangeReaderFunc func(httpRange http_range.Range) (io.ReadCloser, error)
+var _ RangeReadCloserIF = (*RangeReadCloser)(nil)
+
+type RangeReadCloser struct {
+	RangeReader RangeReaderFunc
+	utils.Closers
+}
+
+func (r RangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) {
+	rc, err := r.RangeReader(ctx, httpRange)
+	r.Closers.Add(rc)
+	return rc, err
+}
+
+// type WriterFunc func(w io.Writer) error
+type RangeReaderFunc func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error)

internal/model/file.go

@@ -0,0 +1,25 @@
+package model
+
+import "io"
+
+// File is basic file level accessing interface
+type File interface {
+	io.Reader
+	io.ReaderAt
+	io.Seeker
+	io.Closer
+}
+
+type NopMFileIF interface {
+	io.Reader
+	io.ReaderAt
+	io.Seeker
+}
+type NopMFile struct {
+	NopMFileIF
+}
+
+func (NopMFile) Close() error { return nil }
+func NewNopMFile(r NopMFileIF) File {
+	return NopMFile{r}
+}

internal/model/obj.go

@@ -1,6 +1,8 @@
 package model
 
 import (
+	"github.com/alist-org/alist/v3/pkg/http_range"
+	"github.com/alist-org/alist/v3/pkg/utils"
 	"io"
 	"regexp"
 	"sort"
@@ -20,8 +22,9 @@ type Obj interface {
 	GetSize() int64
 	GetName() string
 	ModTime() time.Time
+	CreateTime() time.Time
 	IsDir() bool
-	//GetHash() (string, string)
+	GetHash() utils.HashInfo
 
 	// The internal information of the driver.
 	// If you want to use it, please understand what it means
@@ -29,14 +32,20 @@ type Obj interface {
 	GetPath() string
 }
 
+// FileStreamer ->check FileStream for more comments
 type FileStreamer interface {
-	io.ReadCloser
+	io.Reader
+	io.Closer
 	Obj
 	GetMimetype() string
-	SetReadCloser(io.ReadCloser)
+	//SetReader(io.Reader)
 	NeedStore() bool
-	GetReadCloser() io.ReadCloser
-	GetOld() Obj
+	GetExist() Obj
+	SetExist(Obj)
+	//for a non-seekable Stream, RangeRead supports peeking some data, and CacheFullInTempFile still works
+	RangeRead(http_range.Range) (io.Reader, error)
+	//for a non-seekable Stream, if Read is called, this function won't work
+	CacheFullInTempFile() (File, error)
 }
 
 type URL interface {
@@ -50,9 +59,6 @@ type Thumb interface {
 type SetPath interface {
 	SetPath(path string)
 }
-type SetHash interface {
-	SetHash(hash string, hashType string)
-}
 
 func SortFiles(objs []Obj, orderBy, orderDirection string) {
 	if orderBy == "" {

internal/model/object.go

@@ -28,9 +28,9 @@ type Object struct {
 	Name     string
 	Size     int64
 	Modified time.Time
+	Ctime    time.Time // file create time
 	IsFolder bool
-	Hash     string
-	HashType string
+	HashInfo utils.HashInfo
 }
 
 func (o *Object) GetName() string {
@@ -44,6 +44,12 @@ func (o *Object) GetSize() int64 {
 func (o *Object) ModTime() time.Time {
 	return o.Modified
 }
+func (o *Object) CreateTime() time.Time {
+	if o.Ctime.IsZero() {
+		return o.ModTime()
+	}
+	return o.Ctime
+}
 
 func (o *Object) IsDir() bool {
 	return o.IsFolder
@@ -61,13 +67,8 @@ func (o *Object) SetPath(path string) {
 	o.Path = path
 }
 
-func (o *Object) SetHash(hash string, hashType string) {
-	o.Hash = hash
-	o.HashType = hashType
-}
-
-func (o *Object) GetHash() (string, string) {
-	return o.Hash, o.HashType
+func (o *Object) GetHash() utils.HashInfo {
+	return o.HashInfo
 }
 
 type Thumbnail struct {

internal/model/stream.go

@@ -1,33 +0,0 @@
-package model
-
-import (
-	"io"
-)
-
-type FileStream struct {
-	Obj
-	io.ReadCloser
-	Mimetype     string
-	WebPutAsTask bool
-	Old          Obj
-}
-
-func (f *FileStream) GetMimetype() string {
-	return f.Mimetype
-}
-
-func (f *FileStream) NeedStore() bool {
-	return f.WebPutAsTask
-}
-
-func (f *FileStream) GetReadCloser() io.ReadCloser {
-	return f.ReadCloser
-}
-
-func (f *FileStream) SetReadCloser(rc io.ReadCloser) {
-	f.ReadCloser = rc
-}
-
-func (f *FileStream) GetOld() Obj {
-	return f.Old
-}

internal/model/user.go

@@ -124,11 +124,11 @@ func (u *User) JoinPath(reqPath string) (string, error) {
 }
 
 func StaticHash(password string) string {
-	return utils.GetSHA256Encode([]byte(fmt.Sprintf("%s-%s", password, StaticHashSalt)))
+	return utils.HashData(utils.SHA256, []byte(fmt.Sprintf("%s-%s", password, StaticHashSalt)))
 }
 
 func HashPwd(static string, salt string) string {
-	return utils.GetSHA256Encode([]byte(fmt.Sprintf("%s-%s", static, salt)))
+	return utils.HashData(utils.SHA256, []byte(fmt.Sprintf("%s-%s", static, salt)))
 }
 
 func TwoHashPwd(password string, salt string) string {