ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: misc improvements about upload/copy/hash (#5045)

Browse Source 
general: add createTime/updateTime support in webdav and some drivers
general: add hash support in some drivers
general: cross-storage rapid-upload support
general: enhance upload to avoid local temp file if possible
general: replace readseekcloser with File interface to speed upstream operations
feat(aliyun_open): same as above
feat(crypt): add hack for 139cloud

Close #4934 
Close #4819 

baidu_netdisk needs to improve the upload code to support rapid-upload
This commit is contained in:
Sean
2023-08-27 21:14:23 +08:00
committed by GitHub
parent 9b765ef696
commit a3748af772
77 changed files with 1731 additions and 615 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
server/common/proxy.go
View File

@ -3,58 +3,35 @@ package common
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"sync"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/net"
"github.com/alist-org/alist/v3/pkg/http_range"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
"io"
"net/http"
"net/url"
)
func HttpClient() *http.Client {
once.Do(func() {
httpClient = base.NewHttpClient()
httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
req.Header.Del("Referer")
return nil
}
})
return httpClient
}
var once sync.Once
var httpClient *http.Client
func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error {
if link.ReadSeekCloser != nil {
if link.MFile != nil {
attachFileName(w, file)
http.ServeContent(w, r, file.GetName(), file.ModTime(), link.ReadSeekCloser)
defer link.ReadSeekCloser.Close()
http.ServeContent(w, r, file.GetName(), file.ModTime(), link.MFile)
defer link.MFile.Close()
return nil
} else if link.RangeReadCloser.RangeReader != nil {
} else if link.RangeReadCloser != nil {
attachFileName(w, file)
net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), link.RangeReadCloser.RangeReader)
net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), link.RangeReadCloser.RangeRead)
defer func() {
if link.RangeReadCloser.Closers != nil {
link.RangeReadCloser.Closers.Close()
}
_ = link.RangeReadCloser.Close()
}()
return nil
} else if link.Concurrency != 0 || link.PartSize != 0 {
attachFileName(w, file)
size := file.GetSize()
//var finalClosers model.Closers
finalClosers := utils.NewClosers()
finalClosers := utils.EmptyClosers()
header := net.ProcessHeader(r.Header, link.Header)
rangeReader := func(httpRange http_range.Range) (io.ReadCloser, error) {
rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) {
down := net.NewDownloader(func(d *net.Downloader) {
d.Concurrency = link.Concurrency
d.PartSize = link.PartSize
@ -65,7 +42,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.
Size: size,
HeaderRef: header,
}
rc, err := down.Download(context.Background(), req)
rc, err := down.Download(ctx, req)
finalClosers.Add(rc)
return rc, err
}
@ -75,7 +52,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.
} else {
//transparent proxy
header := net.ProcessHeader(r.Header, link.Header)
res, err := net.RequestHttp(r.Method, header, link.URL)
res, err := net.RequestHttp(context.Background(), r.Method, header, link.URL)
if err != nil {
return err
}

4
server/handles/down.go
View File

@ -40,13 +40,13 @@ func Down(c *gin.Context) {
common.ErrorResp(c, err, 500)
return
}
if link.ReadSeekCloser != nil {
if link.MFile != nil {
defer func(ReadSeekCloser io.ReadCloser) {
err := ReadSeekCloser.Close()
if err != nil {
log.Errorf("close data error: %s", err)
}
}(link.ReadSeekCloser)
}(link.MFile)
}
c.Header("Referrer-Policy", "no-referrer")
c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate")

4
server/handles/fsmanage.go
View File

@ -331,13 +331,13 @@ func Link(c *gin.Context) {
common.ErrorResp(c, err, 500)
return
}
if link.ReadSeekCloser != nil {
if link.MFile != nil {
defer func(ReadSeekCloser io.ReadCloser) {
err := ReadSeekCloser.Close()
if err != nil {
log.Errorf("close link data error: %v", err)
}
}(link.ReadSeekCloser)
}(link.MFile)
}
common.SuccessResp(c, link)
return

4
server/handles/fsread.go
View File

@ -37,9 +37,11 @@ type ObjResp struct {
Size int64 `json:"size"`
IsDir bool `json:"is_dir"`
Modified time.Time `json:"modified"`
Created time.Time `json:"created"`
Sign string `json:"sign"`
Thumb string `json:"thumb"`
Type int `json:"type"`
HashInfo string `json:"hashinfo"`
}
type FsListResp struct {
@ -313,6 +315,8 @@ func FsGet(c *gin.Context) {
Size: obj.GetSize(),
IsDir: obj.IsDir(),
Modified: obj.ModTime(),
Created: obj.CreateTime(),
HashInfo: obj.GetHash().String(),
Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)),
Type: utils.GetFileType(obj.GetName()),
Thumb: thumb,

28
server/handles/fsup.go
View File

@ -1,6 +1,7 @@
package handles
import (
"github.com/alist-org/alist/v3/internal/stream"
"net/url"
stdpath "path"
"strconv"
@ -33,21 +34,22 @@ func FsStream(c *gin.Context) {
common.ErrorResp(c, err, 400)
return
}
stream := &model.FileStream{
s := &stream.FileStream{
Obj: &model.Object{
Name: name,
Size: size,
Modified: time.Now(),
},
ReadCloser: c.Request.Body,
Reader: c.Request.Body,
Mimetype: c.GetHeader("Content-Type"),
WebPutAsTask: asTask,
}
if asTask {
err = fs.PutAsTask(dir, stream)
err = fs.PutAsTask(dir, s)
} else {
err = fs.PutDirectly(c, dir, stream, true)
err = fs.PutDirectly(c, dir, s, true)
}
defer c.Request.Body.Close()
if err != nil {
common.ErrorResp(c, err, 500)
return
@ -89,21 +91,27 @@ func FsForm(c *gin.Context) {
return
}
dir, name := stdpath.Split(path)
stream := &model.FileStream{
s := stream.FileStream{
Obj: &model.Object{
Name: name,
Size: file.Size,
Modified: time.Now(),
},
ReadCloser: f,
Reader: f,
Mimetype: file.Header.Get("Content-Type"),
WebPutAsTask: false,
}
if asTask {
err = fs.PutAsTask(dir, stream)
} else {
err = fs.PutDirectly(c, dir, stream, true)
ss, err := stream.NewSeekableStream(s, nil)
if err != nil {
common.ErrorResp(c, err, 500)
return
}
if asTask {
err = fs.PutAsTask(dir, ss)
} else {
err = fs.PutDirectly(c, dir, ss, true)
}
defer f.Close()
if err != nil {
common.ErrorResp(c, err, 500)
return

7
server/webdav/prop.go
View File

@ -131,8 +131,8 @@ var liveProps = map[xml.Name]struct {
dir: true,
},
{Space: "DAV:", Local: "creationdate"}: {
findFn: nil,
dir: false,
findFn: findCreationDate,
dir: true,
},
{Space: "DAV:", Local: "getcontentlanguage"}: {
findFn: nil,
@ -383,6 +383,9 @@ func findContentLength(ctx context.Context, ls LockSystem, name string, fi model
func findLastModified(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) {
return fi.ModTime().UTC().Format(http.TimeFormat), nil
}
func findCreationDate(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) {
return fi.CreateTime().UTC().Format(http.TimeFormat), nil
}
// ErrNotImplemented should be returned by optional interfaces if they
// want the original implementation to be used.

29
server/webdav/util.go Normal file
View File

@ -0,0 +1,29 @@
package webdav
import (
log "github.com/sirupsen/logrus"
"net/http"
"strconv"
"time"
)
func (h *Handler) getModTime(r *http.Request) time.Time {
return h.getHeaderTime(r, "X-OC-Mtime")
}
// owncloud/ nextcloud haven't impl this, but we can add the support since rclone may support this soon
func (h *Handler) getCreateTime(r *http.Request) time.Time {
return h.getHeaderTime(r, "X-OC-Ctime")
}
func (h *Handler) getHeaderTime(r *http.Request, header string) time.Time {
hVal := r.Header.Get(header)
if hVal != "" {
modTimeUnix, err := strconv.ParseInt(hVal, 10, 64)
if err == nil {
return time.Unix(modTimeUnix, 0)
}
log.Warnf("getModTime in Webdav, failed to parse %s, %s", header, err)
}
return time.Now()
}

14
server/webdav/webdav.go
View File

@ -8,6 +8,7 @@ package webdav // import "golang.org/x/net/webdav"
import (
"errors"
"fmt"
"github.com/alist-org/alist/v3/internal/stream"
"net/http"
"net/url"
"os"
@ -321,12 +322,13 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
obj := model.Object{
Name: path.Base(reqPath),
Size: r.ContentLength,
Modified: time.Now(),
Modified: h.getModTime(r),
Ctime: h.getCreateTime(r),
}
stream := &model.FileStream{
Obj: &obj,
ReadCloser: r.Body,
Mimetype: r.Header.Get("Content-Type"),
stream := &stream.FileStream{
Obj: &obj,
Reader: r.Body,
Mimetype: r.Header.Get("Content-Type"),
}
if stream.Mimetype == "" {
stream.Mimetype = utils.GetMimeType(reqPath)
@ -336,6 +338,8 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
return http.StatusNotFound, err
}
_ = r.Body.Close()
_ = stream.Close()
// TODO(rost): Returning 405 Method Not Allowed might not be appropriate.
if err != nil {
return http.StatusMethodNotAllowed, err