perf(alias): disabled log on fs call (close #4054)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

build.sh

@@ -59,7 +59,8 @@ BuildDev() {
   mv alist-* dist
   cd dist
   upx -9 ./alist-linux*
-  upx -9 ./alist-windows-amd64.exe
+  cp ./alist-windows-amd64.exe ./alist-windows-amd64-upx.exe
+  upx -9 ./alist-windows-amd64-upx.exe
   find . -type f -print0 | xargs -0 md5sum >md5.txt
   cat md5.txt
 }
@@ -95,7 +96,8 @@ BuildRelease() {
   xgo -out "$appName" -ldflags="$ldflags" -tags=jsoniter .
   # why? Because some target platforms seem to have issues with upx compression
   upx -9 ./alist-linux-amd64
-  upx -9 ./alist-windows-amd64.exe
+  cp ./alist-windows-amd64.exe ./alist-windows-amd64-upx.exe
+  upx -9 ./alist-windows-amd64-upx.exe
   mv alist-* build
 }
 

drivers/123/driver.go

@@ -29,7 +29,6 @@ import (
 type Pan123 struct {
 	model.Storage
 	Addition
-	AccessToken string
 }
 
 func (d *Pan123) Config() driver.Config {
@@ -41,7 +40,8 @@ func (d *Pan123) GetAddition() driver.Additional {
 }
 
 func (d *Pan123) Init(ctx context.Context) error {
-	return d.login()
+	_, err := d.request(UserInfo, http.MethodGet, nil, nil)
+	return err
 }
 
 func (d *Pan123) Drop(ctx context.Context) error {
@@ -77,7 +77,7 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs)
 			"size":      f.Size,
 			"type":      f.Type,
 		}
-		resp, err := d.request("https://www.123pan.com/api/file/download_info", http.MethodPost, func(req *resty.Request) {
+		resp, err := d.request(DownloadInfo, http.MethodPost, func(req *resty.Request) {
 			req.SetBody(data).SetHeaders(headers)
 		}, nil)
 		if err != nil {
@@ -124,7 +124,7 @@ func (d *Pan123) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin
 		"size":         0,
 		"type":         1,
 	}
-	_, err := d.request("https://www.123pan.com/api/file/upload_request", http.MethodPost, func(req *resty.Request) {
+	_, err := d.request(Mkdir, http.MethodPost, func(req *resty.Request) {
 		req.SetBody(data)
 	}, nil)
 	return err
@@ -135,7 +135,7 @@ func (d *Pan123) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
 		"fileIdList":   []base.Json{{"FileId": srcObj.GetID()}},
 		"parentFileId": dstDir.GetID(),
 	}
-	_, err := d.request("https://www.123pan.com/api/file/mod_pid", http.MethodPost, func(req *resty.Request) {
+	_, err := d.request(Move, http.MethodPost, func(req *resty.Request) {
 		req.SetBody(data)
 	}, nil)
 	return err
@@ -147,7 +147,7 @@ func (d *Pan123) Rename(ctx context.Context, srcObj model.Obj, newName string) e
 		"fileId":   srcObj.GetID(),
 		"fileName": newName,
 	}
-	_, err := d.request("https://www.123pan.com/api/file/rename", http.MethodPost, func(req *resty.Request) {
+	_, err := d.request(Rename, http.MethodPost, func(req *resty.Request) {
 		req.SetBody(data)
 	}, nil)
 	return err
@@ -164,7 +164,7 @@ func (d *Pan123) Remove(ctx context.Context, obj model.Obj) error {
 			"operation":         true,
 			"fileTrashInfoList": []File{f},
 		}
-		_, err := d.request("https://www.123pan.com/b/api/file/trash", http.MethodPost, func(req *resty.Request) {
+		_, err := d.request(Trash, http.MethodPost, func(req *resty.Request) {
 			req.SetBody(data)
 		}, nil)
 		return err
@@ -220,7 +220,7 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
 		"type":         0,
 	}
 	var resp UploadResp
-	_, err := d.request("https://www.123pan.com/a/api/file/upload_request", http.MethodPost, func(req *resty.Request) {
+	_, err := d.request(UploadRequest, http.MethodPost, func(req *resty.Request) {
 		req.SetBody(data).SetContext(ctx)
 	}, &resp)
 	if err != nil {
@@ -249,7 +249,7 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
 	if err != nil {
 		return err
 	}
-	_, err = d.request("https://www.123pan.com/api/file/upload_complete", http.MethodPost, func(req *resty.Request) {
+	_, err = d.request(UploadComplete, http.MethodPost, func(req *resty.Request) {
 		req.SetBody(base.Json{
 			"fileId": resp.Data.FileId,
 		}).SetContext(ctx)

drivers/123/meta.go

@@ -8,12 +8,11 @@ import (
 type Addition struct {
 	Username string `json:"username" required:"true"`
 	Password string `json:"password" required:"true"`
+	driver.RootID
 	OrderBy        string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"`
 	OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
-	driver.RootID
-	// define other
 	StreamUpload   bool   `json:"stream_upload"`
-	//Field string `json:"field" type:"select" required:"true" options:"a,b,c" default:"a"`
+	AccessToken    string
 }
 
 var config = driver.Config{

drivers/123/util.go

@@ -14,9 +14,22 @@ import (
 
 // do others that not defined in Driver interface
 
+const (
+	API            = "https://www.123pan.com/b/api"
+	SignIn         = API + "/user/sign_in"
+	UserInfo       = API + "/user/info"
+	FileList       = API + "/file/list/new"
+	DownloadInfo   = API + "/file/download_info"
+	Mkdir          = API + "/file/upload_request"
+	Move           = API + "/file/mod_pid"
+	Rename         = API + "/file/rename"
+	Trash          = API + "/file/trash"
+	UploadRequest  = API + "/file/upload_request"
+	UploadComplete = API + "/file/upload_complete"
+)
+
 func (d *Pan123) login() error {
 	var body base.Json
-	url := "https://www.123pan.com/a/api/user/sign_in"
 	if utils.IsEmailFormat(d.Username) {
 		body = base.Json{
 			"mail":     d.Username,
@@ -30,7 +43,7 @@ func (d *Pan123) login() error {
 		}
 	}
 	res, err := base.RestyClient.R().
-		SetBody(body).Post(url)
+		SetBody(body).Post(SignIn)
 	if err != nil {
 		return err
 	}
@@ -90,7 +103,7 @@ func (d *Pan123) getFiles(parentId string) ([]File, error) {
 			"trashed":        "false",
 			"Page":           strconv.Itoa(page),
 		}
-		_, err := d.request("https://www.123pan.com/api/file/list/new", http.MethodGet, func(req *resty.Request) {
+		_, err := d.request(FileList, http.MethodGet, func(req *resty.Request) {
 			req.SetQueryParams(query)
 		}, &resp)
 		if err != nil {

drivers/alias/util.go

@@ -51,7 +51,7 @@ func (d *Alias) getRootAndPath(path string) (string, string) {
 }
 
 func (d *Alias) get(ctx context.Context, path string, dst, sub string) (model.Obj, error) {
-	obj, err := fs.Get(ctx, stdpath.Join(dst, sub))
+	obj, err := fs.Get(ctx, stdpath.Join(dst, sub), &fs.GetArgs{NoLog: true})
 	if err != nil {
 		return nil, err
 	}
@@ -65,7 +65,7 @@ func (d *Alias) get(ctx context.Context, path string, dst, sub string) (model.Ob
 }
 
 func (d *Alias) list(ctx context.Context, dst, sub string) ([]model.Obj, error) {
-	objs, err := fs.List(ctx, stdpath.Join(dst, sub))
+	objs, err := fs.List(ctx, stdpath.Join(dst, sub), &fs.ListArgs{NoLog: true})
 	// the obj must implement the model.SetPath interface
 	// return objs, err
 	if err != nil {
@@ -83,11 +83,11 @@ func (d *Alias) list(ctx context.Context, dst, sub string) ([]model.Obj, error)
 
 func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs) (*model.Link, error) {
 	reqPath := stdpath.Join(dst, sub)
-	storage, err := fs.GetStorage(reqPath)
+	storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{NoLog: true})
 	if err != nil {
 		return nil, err
 	}
-	_, err = fs.Get(ctx, reqPath)
+	_, err = fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true})
 	if err != nil {
 		return nil, err
 	}

drivers/aliyundrive_open/driver.go

@@ -5,7 +5,7 @@ import (
 	"io"
 	"math"
 	"net/http"
-	"strings"
+	"time"
 
 	"github.com/alist-org/alist/v3/drivers/base"
 	"github.com/alist-org/alist/v3/internal/driver"
@@ -148,11 +148,7 @@ func (d *AliyundriveOpen) Put(ctx context.Context, dstDir model.Obj, stream mode
 	count := 1
 	if stream.GetSize() > DEFAULT {
 		count = int(math.Ceil(float64(stream.GetSize()) / float64(DEFAULT)))
-		partInfoList := make([]base.Json, 0, count)
+		createData["part_info_list"] = makePartInfos(count)
-		for i := 1; i <= count; i++ {
-			partInfoList = append(partInfoList, base.Json{"part_number": i})
-		}
-		createData["part_info_list"] = partInfoList
 	}
 	var createResp CreateResp
 	_, err := d.request("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) {
@@ -162,28 +158,26 @@ func (d *AliyundriveOpen) Put(ctx context.Context, dstDir model.Obj, stream mode
 		return err
 	}
 	// 2. upload
-	for i, partInfo := range createResp.PartInfoList {
+	preTime := time.Now()
+	for i := 1; i <= len(createResp.PartInfoList); i++ {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
 		}
-		uploadUrl := partInfo.UploadUrl
+		err = d.uploadPart(ctx, i, count, utils.NewMultiReadable(io.LimitReader(stream, DEFAULT)), &createResp, true)
-		if d.InternalUpload {
-			//Replace a known public Host with an internal Host
-			uploadUrl = strings.ReplaceAll(uploadUrl, "https://cn-beijing-data.aliyundrive.net/", "http://ccp-bj29-bj-1592982087.oss-cn-beijing-internal.aliyuncs.com/")
-		}
-		req, err := http.NewRequest("PUT", uploadUrl, io.LimitReader(stream, DEFAULT))
 		if err != nil {
 			return err
 		}
-		req = req.WithContext(ctx)
-		res, err := base.HttpClient.Do(req)
-		if err != nil {
-			return err
-		}
-		res.Body.Close()
 		if count > 0 {
 			up(i * 100 / count)
 		}
+		// refresh upload url if 50 minutes passed
+		if time.Since(preTime) > 50*time.Minute {
+			createResp.PartInfoList, err = d.getUploadUrl(count, createResp.FileId, createResp.UploadId)
+			if err != nil {
+				return err
+			}
+			preTime = time.Now()
+		}
 	}
 	// 3. complete
 	_, err = d.request("/adrive/v1.0/openFile/complete", http.MethodPost, func(req *resty.Request) {

drivers/aliyundrive_open/meta.go

@@ -14,7 +14,7 @@ type Addition struct {
 	ClientID       string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"`
 	ClientSecret   string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"`
 	RemoveWay      string `json:"remove_way" required:"true" type:"select" options:"trash,delete"`
-	InternalUpload bool   `json:"internal_upload" help:"If you are using Aliyun ECS in Beijing, you can turn it on to boost the upload speed"`
+	InternalUpload bool   `json:"internal_upload" help:"If you are using Aliyun ECS is located in Beijing, you can turn it on to boost the upload speed"`
 	AccessToken    string
 }
 

drivers/aliyundrive_open/types.go

@@ -45,6 +45,14 @@ func fileToObj(f File) *model.ObjThumb {
 	}
 }
 
+type PartInfo struct {
+	Etag        interface{} `json:"etag"`
+	PartNumber  int         `json:"part_number"`
+	PartSize    interface{} `json:"part_size"`
+	UploadUrl   string      `json:"upload_url"`
+	ContentType string      `json:"content_type"`
+}
+
 type CreateResp struct {
 	//Type         string `json:"type"`
 	//ParentFileId string `json:"parent_file_id"`
@@ -57,11 +65,5 @@ type CreateResp struct {
 	UploadId string `json:"upload_id"`
 	//Location     string `json:"location"`
 	RapidUpload  bool       `json:"rapid_upload"`
-	PartInfoList []struct {
+	PartInfoList []PartInfo `json:"part_info_list"`
-		Etag        interface{} `json:"etag"`
-		PartNumber  int         `json:"part_number"`
-		PartSize    interface{} `json:"part_size"`
-		UploadUrl   string      `json:"upload_url"`
-		ContentType string      `json:"content_type"`
-	} `json:"part_info_list"`
 }

drivers/aliyundrive_open/util.go

@@ -1,12 +1,15 @@
 package aliyundrive_open
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/alist-org/alist/v3/drivers/base"
 	"github.com/alist-org/alist/v3/internal/op"
+	"github.com/alist-org/alist/v3/pkg/utils"
 	"github.com/go-resty/resty/v2"
 )
 
@@ -106,3 +109,59 @@ func (d *AliyundriveOpen) getFiles(fileId string) ([]File, error) {
 	}
 	return res, nil
 }
+
+func makePartInfos(size int) []base.Json {
+	partInfoList := make([]base.Json, size)
+	for i := 0; i < size; i++ {
+		partInfoList[i] = base.Json{"part_number": 1 + i}
+	}
+	return partInfoList
+}
+
+func (d *AliyundriveOpen) getUploadUrl(count int, fileId, uploadId string) ([]PartInfo, error) {
+	partInfoList := makePartInfos(count)
+	var resp CreateResp
+	_, err := d.request("/adrive/v1.0/openFile/getUploadUrl", http.MethodPost, func(req *resty.Request) {
+		req.SetBody(base.Json{
+			"drive_id":       d.DriveId,
+			"file_id":        fileId,
+			"part_info_list": partInfoList,
+			"upload_id":      uploadId,
+		}).SetResult(&resp)
+	})
+	return resp.PartInfoList, err
+}
+
+func (d *AliyundriveOpen) uploadPart(ctx context.Context, i, count int, reader *utils.MultiReadable, resp *CreateResp, retry bool) error {
+	partInfo := resp.PartInfoList[i-1]
+	uploadUrl := partInfo.UploadUrl
+	if d.InternalUpload {
+		uploadUrl = strings.ReplaceAll(uploadUrl, "https://cn-beijing-data.aliyundrive.net/", "http://ccp-bj29-bj-1592982087.oss-cn-beijing-internal.aliyuncs.com/")
+	}
+	req, err := http.NewRequest("PUT", uploadUrl, reader)
+	if err != nil {
+		return err
+	}
+	req = req.WithContext(ctx)
+	res, err := base.HttpClient.Do(req)
+	if err != nil {
+		if retry {
+			reader.Reset()
+			return d.uploadPart(ctx, i, count, reader, resp, false)
+		}
+		return err
+	}
+	res.Body.Close()
+	if retry && res.StatusCode == http.StatusForbidden {
+		resp.PartInfoList, err = d.getUploadUrl(count, resp.FileId, resp.UploadId)
+		if err != nil {
+			return err
+		}
+		reader.Reset()
+		return d.uploadPart(ctx, i, count, reader, resp, false)
+	}
+	if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusConflict {
+		return fmt.Errorf("upload status: %d", res.StatusCode)
+	}
+	return nil
+}

drivers/s3/driver.go

@@ -128,6 +128,9 @@ func (d *S3) Remove(ctx context.Context, obj model.Obj) error {
 
 func (d *S3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
 	uploader := s3manager.NewUploader(d.Session)
+	if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize {
+		uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1)
+	}
 	key := getKey(stdpath.Join(dstDir.GetPath(), stream.GetName()), false)
 	log.Debugln("key:", key)
 	input := &s3manager.UploadInput{

drivers/trainbit/util.go

@@ -113,10 +113,10 @@ func parseRawFileObject(rawObject []any) ([]model.Obj, error) {
 		}
 		isFolder := int64(object["ty"].(float64)) == 1
 		var name string
-		if isFolder {
+		if object["ext"].(string) != "" {
-			name = object["name"].(string)
+			name = strings.Join([]string{object["name"].(string), object["ext"].(string)}, ".")
 		} else {
-			name = object["name"].(string) + object["ext"].(string)
+			name = object["name"].(string)
 		}
 		modified, err := time.Parse("2006/01/02 15:04:05", object["modified"].(string))
 		if err != nil {

go.mod

@@ -26,10 +26,10 @@ require (
 	github.com/pkg/sftp v1.13.5
 	github.com/pquerna/otp v1.4.0
 	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v1.6.1
+	github.com/spf13/cobra v1.7.0
 	github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca
 	github.com/u2takey/ffmpeg-go v0.4.1
-	github.com/upyun/go-sdk/v3 v3.0.3
+	github.com/upyun/go-sdk/v3 v3.0.4
 	github.com/winfsp/cgofuse v1.5.0
 	golang.org/x/crypto v0.7.0
 	golang.org/x/image v0.6.0
@@ -79,7 +79,7 @@ require (
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx/v5 v5.3.0 // indirect

go.sum

@@ -129,8 +129,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI=
 github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
-github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
-github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
@@ -214,8 +214,8 @@ github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -244,8 +244,8 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6
 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
 github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU=
 github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
-github.com/upyun/go-sdk/v3 v3.0.3 h1:2wUkNk2fyJReMYHMvJyav050D83rYwSjN7mEPR0Pp8Q=
+github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q=
-github.com/upyun/go-sdk/v3 v3.0.3/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E=
+github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E=
 github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
 github.com/winfsp/cgofuse v1.5.0 h1:MsBP7Mi/LiJf/7/F3O/7HjjR009ds6KCdqXzKpZSWxI=
 github.com/winfsp/cgofuse v1.5.0/go.mod h1:h3awhoUOcn2VYVKCwDaYxSLlZwnyK+A8KaDoLUp2lbU=

internal/fs/fs.go

@@ -13,8 +13,13 @@ import (
 // So, the purpose of this package is to convert mount path to actual path
 // then pass the actual path to the op package
 
-func List(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error) {
+type ListArgs struct {
-	res, err := list(ctx, path, refresh...)
+	Refresh bool
+	NoLog   bool
+}
+
+func List(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) {
+	res, err := list(ctx, path, args)
 	if err != nil {
 		log.Errorf("failed list %s: %+v", path, err)
 		return nil, err
@@ -22,9 +27,13 @@ func List(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error
 	return res, nil
 }
 
-func Get(ctx context.Context, path string) (model.Obj, error) {
+type GetArgs struct {
+	NoLog bool
+}
+
+func Get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) {
 	res, err := get(ctx, path)
-	if err != nil {
+	if err != nil && !args.NoLog {
 		log.Errorf("failed get %s: %+v", path, err)
 		return nil, err
 	}
@@ -96,9 +105,13 @@ func PutAsTask(dstDirPath string, file *model.FileStream) error {
 	return err
 }
 
-func GetStorage(path string) (driver.Driver, error) {
+type GetStoragesArgs struct {
+	NoLog bool
+}
+
+func GetStorage(path string, args *GetStoragesArgs) (driver.Driver, error) {
 	storageDriver, _, err := op.GetStorageAndActualPath(path)
-	if err != nil {
+	if err != nil && !args.NoLog {
 		return nil, err
 	}
 	return storageDriver, nil

internal/fs/list.go

@@ -11,7 +11,7 @@ import (
 )
 
 // List files
-func list(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error) {
+func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) {
 	meta := ctx.Value("meta").(*model.Meta)
 	user := ctx.Value("user").(*model.User)
 	virtualFiles := op.GetStorageVirtualFilesByPath(path)
@@ -24,7 +24,7 @@ func list(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error
 	if storage != nil {
 		_objs, err = op.List(ctx, storage, actualPath, model.ListArgs{
 			ReqPath: path,
-		}, refresh...)
+		}, args.Refresh)
 		if err != nil {
 			log.Errorf("%+v", err)
 			if len(virtualFiles) == 0 {

internal/fs/walk.go

@@ -28,7 +28,7 @@ func WalkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn
 	}
 	meta, _ := op.GetNearestMeta(name)
 	// Read directory names.
-	objs, err := List(context.WithValue(ctx, "meta", meta), name)
+	objs, err := List(context.WithValue(ctx, "meta", meta), name, &ListArgs{})
 	if err != nil {
 		return walkFnErr
 	}

internal/search/build.go

@@ -141,7 +141,7 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
 			})
 			return nil
 		}
-		fi, err = fs.Get(ctx, indexPath)
+		fi, err = fs.Get(ctx, indexPath, &fs.GetArgs{})
 		if err != nil {
 			return err
 		}

pkg/gowebdav/client.go

@@ -343,9 +343,9 @@ func (c *Client) Read(path string) ([]byte, error) {
 }
 
 func (c *Client) Link(path string) (string, http.Header, error) {
-	method := "HEAD"
+	method := "GET"
-	url := PathEscape(Join(c.root, path))
+	u := PathEscape(Join(c.root, path))
-	r, err := http.NewRequest(method, url, nil)
+	r, err := http.NewRequest(method, u, nil)
 
 	if err != nil {
 		return "", nil, newPathErrorErr("Link", path, err)
@@ -366,31 +366,6 @@ func (c *Client) Link(path string) (string, http.Header, error) {
 	if c.interceptor != nil {
 		c.interceptor(method, r)
 	}
-
-	rs, err := c.c.Do(r)
-	if err != nil {
-		return "", nil, newPathErrorErr("Link", path, err)
-	}
-
-	if rs.StatusCode == 401 {
-		wwwAuthenticateHeader := strings.ToLower(rs.Header.Get("Www-Authenticate"))
-		if strings.Contains(wwwAuthenticateHeader, "digest") {
-			c.authMutex.Lock()
-			c.auth = &DigestAuth{auth.User(), auth.Pass(), digestParts(rs)}
-			c.auth.Authorize(r, method, path)
-			c.authMutex.Unlock()
-		} else if strings.Contains(wwwAuthenticateHeader, "basic") {
-			c.authMutex.Lock()
-			c.auth = &BasicAuth{auth.User(), auth.Pass()}
-			c.auth.Authorize(r, method, path)
-			c.authMutex.Unlock()
-		} else {
-			return "", nil, newPathError("Authorize", c.root, rs.StatusCode)
-		}
-	} else if rs.StatusCode > 400 {
-		return "", nil, newPathError("Authorize", path, rs.StatusCode)
-	}
-
 	return r.URL.String(), r.Header, nil
 }
 

pkg/utils/io.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"bytes"
 	"context"
 	"io"
 )
@@ -91,3 +92,46 @@ func NewReadCloser(reader io.Reader, close CloseFunc) io.ReadCloser {
 func NewLimitReadCloser(reader io.Reader, close CloseFunc, limit int64) io.ReadCloser {
 	return NewReadCloser(io.LimitReader(reader, limit), close)
 }
+
+type MultiReadable struct {
+	originReader io.Reader
+	reader       io.Reader
+	cache        *bytes.Buffer
+}
+
+func NewMultiReadable(reader io.Reader) *MultiReadable {
+	return &MultiReadable{
+		originReader: reader,
+		reader:       reader,
+	}
+}
+
+func (mr *MultiReadable) Read(p []byte) (int, error) {
+	n, err := mr.reader.Read(p)
+	if _, ok := mr.reader.(io.Seeker); !ok && n > 0 {
+		if mr.cache == nil {
+			mr.cache = &bytes.Buffer{}
+		}
+		mr.cache.Write(p[:n])
+	}
+	return n, err
+}
+
+func (mr *MultiReadable) Reset() error {
+	if seeker, ok := mr.reader.(io.Seeker); ok {
+		_, err := seeker.Seek(0, io.SeekStart)
+		return err
+	}
+	if mr.cache != nil && mr.cache.Len() > 0 {
+		mr.reader = io.MultiReader(mr.cache, mr.reader)
+		mr.cache = nil
+	}
+	return nil
+}
+
+func (mr *MultiReadable) Close() error {
+	if closer, ok := mr.originReader.(io.Closer); ok {
+		return closer.Close()
+	}
+	return nil
+}

server/common/proxy.go

@@ -16,7 +16,15 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-var HttpClient = &http.Client{}
+var HttpClient = &http.Client{
+	CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		if len(via) >= 10 {
+			return errors.New("stopped after 10 redirects")
+		}
+		req.Header.Del("Referer")
+		return nil
+	},
+}
 
 func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error {
 	// read data with native

server/handles/down.go

@@ -21,7 +21,7 @@ import (
 func Down(c *gin.Context) {
 	rawPath := c.MustGet("path").(string)
 	filename := stdpath.Base(rawPath)
-	storage, err := fs.GetStorage(rawPath)
+	storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
@@ -65,7 +65,7 @@ func Down(c *gin.Context) {
 func Proxy(c *gin.Context) {
 	rawPath := c.MustGet("path").(string)
 	filename := stdpath.Base(rawPath)
-	storage, err := fs.GetStorage(rawPath)
+	storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return

server/handles/fsmanage.go

@@ -133,7 +133,7 @@ func FsRecursiveMove(c *gin.Context) {
 	}
 	c.Set("meta", meta)
 
-	rootFiles, err := fs.List(c, srcDir, false)
+	rootFiles, err := fs.List(c, srcDir, &fs.ListArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
@@ -154,7 +154,7 @@ func FsRecursiveMove(c *gin.Context) {
 		if movingFile.IsDir() {
 			// directory, recursive move
 			subFilePath := movingFilePath
-			subFiles, err := fs.List(c, subFilePath, true)
+			subFiles, err := fs.List(c, subFilePath, &fs.ListArgs{Refresh: true})
 			if err != nil {
 				common.ErrorResp(c, err, 500)
 				return
@@ -293,7 +293,7 @@ func FsRegexRename(c *gin.Context) {
 		return
 	}
 
-	files, err := fs.List(c, reqPath, false)
+	files, err := fs.List(c, reqPath, &fs.ListArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
@@ -362,7 +362,7 @@ func Link(c *gin.Context) {
 	//rawPath := stdpath.Join(user.BasePath, req.Path)
 	// why need not join base_path? because it's always the full path
 	rawPath := req.Path
-	storage, err := fs.GetStorage(rawPath)
+	storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return

server/handles/fsread.go

@@ -6,10 +6,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/alist-org/alist/v3/internal/conf"
 	"github.com/alist-org/alist/v3/internal/errs"
 	"github.com/alist-org/alist/v3/internal/fs"
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/internal/op"
+	"github.com/alist-org/alist/v3/internal/setting"
 	"github.com/alist-org/alist/v3/internal/sign"
 	"github.com/alist-org/alist/v3/pkg/utils"
 	"github.com/alist-org/alist/v3/server/common"
@@ -77,14 +79,14 @@ func FsList(c *gin.Context) {
 		common.ErrorStrResp(c, "Refresh without permission", 403)
 		return
 	}
-	objs, err := fs.List(c, reqPath, req.Refresh)
+	objs, err := fs.List(c, reqPath, &fs.ListArgs{Refresh: req.Refresh})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
 	}
 	total, objs := pagination(objs, &req.PageReq)
 	provider := "unknown"
-	storage, err := fs.GetStorage(reqPath)
+	storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{})
 	if err == nil {
 		provider = storage.GetStorage().Driver
 	}
@@ -130,7 +132,7 @@ func FsDirs(c *gin.Context) {
 		common.ErrorStrResp(c, "password is incorrect or you have no permission", 403)
 		return
 	}
-	objs, err := fs.List(c, reqPath)
+	objs, err := fs.List(c, reqPath, &fs.ListArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
@@ -245,14 +247,14 @@ func FsGet(c *gin.Context) {
 		common.ErrorStrResp(c, "password is incorrect or you have no permission", 403)
 		return
 	}
-	obj, err := fs.Get(c, reqPath)
+	obj, err := fs.Get(c, reqPath, &fs.GetArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
 	}
 	var rawURL string
 
-	storage, err := fs.GetStorage(reqPath)
+	storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{})
 	provider := "unknown"
 	if err == nil {
 		provider = storage.Config().Name
@@ -264,7 +266,7 @@ func FsGet(c *gin.Context) {
 		}
 		if storage.Config().MustProxy() || storage.GetStorage().WebProxy {
 			query := ""
-			if isEncrypt(meta, reqPath) {
+			if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) {
 				query = "?sign=" + sign.Sign(reqPath)
 			}
 			if storage.GetStorage().DownProxyUrl != "" {
@@ -295,7 +297,7 @@ func FsGet(c *gin.Context) {
 	}
 	var related []model.Obj
 	parentPath := stdpath.Dir(reqPath)
-	sameLevelFiles, err := fs.List(c, parentPath)
+	sameLevelFiles, err := fs.List(c, parentPath, &fs.ListArgs{})
 	if err == nil {
 		related = filterRelated(sameLevelFiles, obj)
 	}

server/handles/fsup.go

@@ -69,7 +69,7 @@ func FsForm(c *gin.Context) {
 		common.ErrorResp(c, err, 403)
 		return
 	}
-	storage, err := fs.GetStorage(path)
+	storage, err := fs.GetStorage(path, &fs.GetStoragesArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 400)
 		return

server/webdav/file.go

@@ -84,7 +84,7 @@ func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn
 	}
 	meta, _ := op.GetNearestMeta(name)
 	// Read directory names.
-	objs, err := fs.List(context.WithValue(ctx, "meta", meta), name)
+	objs, err := fs.List(context.WithValue(ctx, "meta", meta), name, &fs.ListArgs{})
 	//f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0)
 	//if err != nil {
 	//	return walkFn(name, info, err)

server/webdav/webdav.go

@@ -188,7 +188,7 @@ func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status
 		return 403, err
 	}
 	allow := "OPTIONS, LOCK, PUT, MKCOL"
-	if fi, err := fs.Get(ctx, reqPath); err == nil {
+	if fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err == nil {
 		if fi.IsDir() {
 			allow = "OPTIONS, LOCK, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND"
 		} else {
@@ -215,7 +215,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta
 	if err != nil {
 		return 403, err
 	}
-	fi, err := fs.Get(ctx, reqPath)
+	fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
 	if err != nil {
 		return http.StatusNotFound, err
 	}
@@ -228,7 +228,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta
 	}
 	w.Header().Set("ETag", etag)
 	// Let ServeContent determine the Content-Type header.
-	storage, _ := fs.GetStorage(reqPath)
+	storage, _ := fs.GetStorage(reqPath, &fs.GetStoragesArgs{})
 	downProxyUrl := storage.GetStorage().DownProxyUrl
 	if storage.GetStorage().WebdavNative() || (storage.GetStorage().WebdavProxy() && downProxyUrl == "") {
 		link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{Header: r.Header})
@@ -278,7 +278,7 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i
 	// "godoc os RemoveAll" says that "If the path does not exist, RemoveAll
 	// returns nil (no error)." WebDAV semantics are that it should return a
 	// "404 Not Found". We therefore have to Stat before we RemoveAll.
-	if _, err := fs.Get(ctx, reqPath); err != nil {
+	if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil {
 		if errs.IsObjectNotFound(err) {
 			return http.StatusNotFound, err
 		}
@@ -331,7 +331,7 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
 	if err != nil {
 		return http.StatusMethodNotAllowed, err
 	}
-	fi, err := fs.Get(ctx, reqPath)
+	fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
 	if err != nil {
 		fi = &obj
 	}
@@ -591,7 +591,7 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status
 	if err != nil {
 		return 403, err
 	}
-	fi, err := fs.Get(ctx, reqPath)
+	fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
 	if err != nil {
 		if errs.IsObjectNotFound(err) {
 			return http.StatusNotFound, err
@@ -670,7 +670,7 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu
 	if err != nil {
 		return 403, err
 	}
-	if _, err := fs.Get(ctx, reqPath); err != nil {
+	if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil {
 		if errs.IsObjectNotFound(err) {
 			return http.StatusNotFound, err
 		}