ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: misc improvements about upload/copy/hash (#5045)

Browse Source 
general: add createTime/updateTime support in webdav and some drivers
general: add hash support in some drivers
general: cross-storage rapid-upload support
general: enhance upload to avoid local temp file if possible
general: replace readseekcloser with File interface to speed upstream operations
feat(aliyun_open): same as above
feat(crypt): add hack for 139cloud

Close #4934 
Close #4819 

baidu_netdisk needs to improve the upload code to support rapid-upload
This commit is contained in:
Sean
2023-08-27 21:14:23 +08:00
committed by GitHub
parent 9b765ef696
commit a3748af772
77 changed files with 1731 additions and 615 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
internal/fs/copy.go
View File

@ -10,6 +10,7 @@ import (
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/stream"
"github.com/alist-org/alist/v3/pkg/task"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
@ -94,9 +95,14 @@ func copyFileBetween2Storages(tsk *task.Task[uint64], srcStorage, dstStorage dri
if err != nil {
return errors.WithMessagef(err, "failed get [%s] link", srcFilePath)
}
stream, err := getFileStreamFromLink(tsk.Ctx, srcFile, link)
fs := stream.FileStream{
Obj: srcFile,
Ctx: tsk.Ctx,
}
// any link provided is seekable
ss, err := stream.NewSeekableStream(fs, link)
if err != nil {
return errors.WithMessagef(err, "failed get [%s] stream", srcFilePath)
}
return op.Put(tsk.Ctx, dstStorage, dstDirPath, stream, tsk.SetProgress, true)
return op.Put(tsk.Ctx, dstStorage, dstDirPath, ss, tsk.SetProgress, true)
}

5
internal/fs/fs.go
View File

@ -2,7 +2,6 @@ package fs
import (
"context"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
@ -93,7 +92,7 @@ func Remove(ctx context.Context, path string) error {
return err
}
func PutDirectly(ctx context.Context, dstDirPath string, file *model.FileStream, lazyCache ...bool) error {
func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, lazyCache ...bool) error {
err := putDirectly(ctx, dstDirPath, file, lazyCache...)
if err != nil {
log.Errorf("failed put %s: %+v", dstDirPath, err)
@ -101,7 +100,7 @@ func PutDirectly(ctx context.Context, dstDirPath string, file *model.FileStream,
return err
}
func PutAsTask(dstDirPath string, file *model.FileStream) error {
func PutAsTask(dstDirPath string, file model.FileStreamer) error {
err := putAsTask(dstDirPath, file)
if err != nil {
log.Errorf("failed put %s: %+v", dstDirPath, err)

12
internal/fs/put.go
View File

@ -3,13 +3,12 @@ package fs
import (
"context"
"fmt"
"github.com/alist-org/alist/v3/internal/model"
"sync/atomic"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/task"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
)
@ -18,7 +17,7 @@ var UploadTaskManager = task.NewTaskManager(3, func(tid *uint64) {
})
// putAsTask add as a put task and return immediately
func putAsTask(dstDirPath string, file *model.FileStream) error {
func putAsTask(dstDirPath string, file model.FileStreamer) error {
storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed get storage")
@ -27,11 +26,12 @@ func putAsTask(dstDirPath string, file *model.FileStream) error {
return errors.WithStack(errs.UploadNotSupported)
}
if file.NeedStore() {
tempFile, err := utils.CreateTempFile(file, file.GetSize())
_, err := file.CacheFullInTempFile()
if err != nil {
return errors.Wrapf(err, "failed to create temp file")
}
file.SetReadCloser(tempFile)
//file.SetReader(tempFile)
//file.SetTmpFile(tempFile)
}
UploadTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{
Name: fmt.Sprintf("upload %s to [%s](%s)", file.GetName(), storage.GetStorage().MountPath, dstDirActualPath),
@ -43,7 +43,7 @@ func putAsTask(dstDirPath string, file *model.FileStream) error {
}
// putDirect put the file and return after finish
func putDirectly(ctx context.Context, dstDirPath string, file *model.FileStream, lazyCache ...bool) error {
func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, lazyCache ...bool) error {
storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath)
if err != nil {
return errors.WithMessage(err, "failed get storage")

73
internal/fs/util.go
View File

@ -1,73 +0,0 @@
package fs
import (
"context"
"io"
"net/http"
"strings"
"github.com/alist-org/alist/v3/internal/net"
"github.com/alist-org/alist/v3/pkg/http_range"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/alist-org/alist/v3/server/common"
"github.com/pkg/errors"
)
func getFileStreamFromLink(ctx context.Context, file model.Obj, link *model.Link) (*model.FileStream, error) {
var rc io.ReadCloser
var err error
mimetype := utils.GetMimeType(file.GetName())
if link.RangeReadCloser.RangeReader != nil {
rc, err = link.RangeReadCloser.RangeReader(http_range.Range{Length: -1})
if err != nil {
return nil, err
}
} else if link.ReadSeekCloser != nil {
rc = link.ReadSeekCloser
} else if link.Concurrency != 0 || link.PartSize != 0 {
down := net.NewDownloader(func(d *net.Downloader) {
d.Concurrency = link.Concurrency
d.PartSize = link.PartSize
})
req := &net.HttpRequestParams{
URL: link.URL,
Range: http_range.Range{Length: -1},
Size: file.GetSize(),
HeaderRef: link.Header,
}
rc, err = down.Download(ctx, req)
if err != nil {
return nil, err
}
} else {
//TODO: add accelerator
req, err := http.NewRequest(http.MethodGet, link.URL, nil)
if err != nil {
return nil, errors.Wrapf(err, "failed to create request for %s", link.URL)
}
for h, val := range link.Header {
req.Header[h] = val
}
res, err := common.HttpClient().Do(req)
if err != nil {
return nil, errors.Wrapf(err, "failed to get response for %s", link.URL)
}
mt := res.Header.Get("Content-Type")
if mt != "" && strings.ToLower(mt) != "application/octet-stream" {
mimetype = mt
}
rc = res.Body
}
// if can't get mimetype, use default application/octet-stream
if mimetype == "" {
mimetype = "application/octet-stream"
}
stream := &model.FileStream{
Obj: file,
ReadCloser: rc,
Mimetype: mimetype,
}
return stream, nil
}