fix(123): adapt new upload method (close #4141)

* 增加清理空文件夹API

* 修复嵌套文件夹删除Bug

 Author:    varg247 <varg247@gmail.com>

---------

Co-authored-by: varg247 <varg247@qq.com>

.github/workflows/auto_lang.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         platform: [ ubuntu-latest ]
-        go-version: [ 1.19 ]
+        go-version: [ '1.20' ]
     name: auto generate lang.json
     runs-on: ${{ matrix.platform }}
     steps:

.github/workflows/build.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest]
-        go-version: [1.19]
+        go-version: [ '1.20' ]
     name: Build
     runs-on: ${{ matrix.platform }}
     steps:

.github/workflows/release.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       matrix:
         platform: [ ubuntu-latest ]
-        go-version: [ 1.19 ]
+        go-version: [ '1.20' ]
     name: Release
     runs-on: ${{ matrix.platform }}
     steps:

Dockerfile

@@ -1,11 +1,11 @@
-FROM alpine:edge as builder
+FROM alpine:3.17 as builder
 LABEL stage=go-builder
 WORKDIR /app/
 COPY ./ ./
 RUN apk add --no-cache bash curl gcc git go musl-dev; \
     bash build.sh release docker
 
-FROM alpine:edge
+FROM alpine:3.17
 LABEL MAINTAINER="i@nn.ci"
 VOLUME /opt/alist/data/
 WORKDIR /opt/alist/

build.sh

@@ -59,7 +59,8 @@ BuildDev() {
   mv alist-* dist
   cd dist
   upx -9 ./alist-linux*
-  upx -9 ./alist-windows-amd64.exe
+  cp ./alist-windows-amd64.exe ./alist-windows-amd64-upx.exe
+  upx -9 ./alist-windows-amd64-upx.exe
   find . -type f -print0 | xargs -0 md5sum >md5.txt
   cat md5.txt
 }
@@ -95,7 +96,8 @@ BuildRelease() {
   xgo -out "$appName" -ldflags="$ldflags" -tags=jsoniter .
   # why? Because some target platforms seem to have issues with upx compression
   upx -9 ./alist-linux-amd64
-  upx -9 ./alist-windows-amd64.exe
+  cp ./alist-windows-amd64.exe ./alist-windows-amd64-upx.exe
+  upx -9 ./alist-windows-amd64-upx.exe
   mv alist-* build
 }
 

drivers/123/driver.go

@@ -29,7 +29,6 @@ import (
 type Pan123 struct {
 	model.Storage
 	Addition
-	AccessToken string
 }
 
 func (d *Pan123) Config() driver.Config {
@@ -41,7 +40,8 @@ func (d *Pan123) GetAddition() driver.Additional {
 }
 
 func (d *Pan123) Init(ctx context.Context) error {
-	return d.login()
+	_, err := d.request(UserInfo, http.MethodGet, nil, nil)
+	return err
 }
 
 func (d *Pan123) Drop(ctx context.Context) error {
@@ -77,7 +77,7 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs)
 			"size":      f.Size,
 			"type":      f.Type,
 		}
-		resp, err := d.request("https://www.123pan.com/api/file/download_info", http.MethodPost, func(req *resty.Request) {
+		resp, err := d.request(DownloadInfo, http.MethodPost, func(req *resty.Request) {
 			req.SetBody(data).SetHeaders(headers)
 		}, nil)
 		if err != nil {
@@ -124,7 +124,7 @@ func (d *Pan123) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin
 		"size":         0,
 		"type":         1,
 	}
-	_, err := d.request("https://www.123pan.com/api/file/upload_request", http.MethodPost, func(req *resty.Request) {
+	_, err := d.request(Mkdir, http.MethodPost, func(req *resty.Request) {
 		req.SetBody(data)
 	}, nil)
 	return err
@@ -135,7 +135,7 @@ func (d *Pan123) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
 		"fileIdList":   []base.Json{{"FileId": srcObj.GetID()}},
 		"parentFileId": dstDir.GetID(),
 	}
-	_, err := d.request("https://www.123pan.com/api/file/mod_pid", http.MethodPost, func(req *resty.Request) {
+	_, err := d.request(Move, http.MethodPost, func(req *resty.Request) {
 		req.SetBody(data)
 	}, nil)
 	return err
@@ -147,7 +147,7 @@ func (d *Pan123) Rename(ctx context.Context, srcObj model.Obj, newName string) e
 		"fileId":   srcObj.GetID(),
 		"fileName": newName,
 	}
-	_, err := d.request("https://www.123pan.com/api/file/rename", http.MethodPost, func(req *resty.Request) {
+	_, err := d.request(Rename, http.MethodPost, func(req *resty.Request) {
 		req.SetBody(data)
 	}, nil)
 	return err
@@ -164,7 +164,7 @@ func (d *Pan123) Remove(ctx context.Context, obj model.Obj) error {
 			"operation":         true,
 			"fileTrashInfoList": []File{f},
 		}
-		_, err := d.request("https://www.123pan.com/b/api/file/trash", http.MethodPost, func(req *resty.Request) {
+		_, err := d.request(Trash, http.MethodPost, func(req *resty.Request) {
 			req.SetBody(data)
 		}, nil)
 		return err
@@ -220,36 +220,41 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
 		"type":         0,
 	}
 	var resp UploadResp
-	_, err := d.request("https://www.123pan.com/a/api/file/upload_request", http.MethodPost, func(req *resty.Request) {
+	res, err := d.request(UploadRequest, http.MethodPost, func(req *resty.Request) {
 		req.SetBody(data).SetContext(ctx)
 	}, &resp)
 	if err != nil {
 		return err
 	}
+	log.Debugln("upload request res: ", string(res))
 	if resp.Data.Reuse || resp.Data.Key == "" {
 		return nil
 	}
-	cfg := &aws.Config{
-		Credentials:      credentials.NewStaticCredentials(resp.Data.AccessKeyId, resp.Data.SecretAccessKey, resp.Data.SessionToken),
-		Region:           aws.String("123pan"),
-		Endpoint:         aws.String("file.123pan.com"),
-		S3ForcePathStyle: aws.Bool(true),
+	if resp.Data.AccessKeyId == "" || resp.Data.SecretAccessKey == "" || resp.Data.SessionToken == "" {
+		err = d.newUpload(ctx, &resp, stream, uploadFile, up)
+	} else {
+		cfg := &aws.Config{
+			Credentials:      credentials.NewStaticCredentials(resp.Data.AccessKeyId, resp.Data.SecretAccessKey, resp.Data.SessionToken),
+			Region:           aws.String("123pan"),
+			Endpoint:         aws.String(resp.Data.EndPoint),
+			S3ForcePathStyle: aws.Bool(true),
+		}
+		s, err := session.NewSession(cfg)
+		if err != nil {
+			return err
+		}
+		uploader := s3manager.NewUploader(s)
+		input := &s3manager.UploadInput{
+			Bucket: &resp.Data.Bucket,
+			Key:    &resp.Data.Key,
+			Body:   uploadFile,
+		}
+		_, err = uploader.UploadWithContext(ctx, input)
 	}
-	s, err := session.NewSession(cfg)
 	if err != nil {
 		return err
 	}
-	uploader := s3manager.NewUploader(s)
-	input := &s3manager.UploadInput{
-		Bucket: &resp.Data.Bucket,
-		Key:    &resp.Data.Key,
-		Body:   uploadFile,
-	}
-	_, err = uploader.UploadWithContext(ctx, input)
-	if err != nil {
-		return err
-	}
-	_, err = d.request("https://www.123pan.com/api/file/upload_complete", http.MethodPost, func(req *resty.Request) {
+	_, err = d.request(UploadComplete, http.MethodPost, func(req *resty.Request) {
 		req.SetBody(base.Json{
 			"fileId": resp.Data.FileId,
 		}).SetContext(ctx)

drivers/123/meta.go

@@ -6,14 +6,13 @@ import (
 )
 
 type Addition struct {
-	Username       string `json:"username" required:"true"`
-	Password       string `json:"password" required:"true"`
+	Username string `json:"username" required:"true"`
+	Password string `json:"password" required:"true"`
+	driver.RootID
 	OrderBy        string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"`
 	OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
-	driver.RootID
-	// define other
-	StreamUpload bool `json:"stream_upload"`
-	//Field string `json:"field" type:"select" required:"true" options:"a,b,c" default:"a"`
+	StreamUpload   bool   `json:"stream_upload"`
+	AccessToken    string
 }
 
 var config = driver.Config{

drivers/123/types.go

@@ -74,5 +74,14 @@ type UploadResp struct {
 		SessionToken    string `json:"SessionToken"`
 		FileId          int64  `json:"FileId"`
 		Reuse           bool   `json:"Reuse"`
+		EndPoint        string `json:"EndPoint"`
+		StorageNode     string `json:"StorageNode"`
+		UploadId        string `json:"UploadId"`
+	} `json:"data"`
+}
+
+type S3PreSignedURLs struct {
+	Data struct {
+		PreSignedUrls map[string]string `json:"presignedUrls"`
 	} `json:"data"`
 }

drivers/123/upload.go

@@ -0,0 +1,127 @@
+package _123
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"strconv"
+
+	"github.com/alist-org/alist/v3/drivers/base"
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/go-resty/resty/v2"
+)
+
+func (d *Pan123) getS3PreSignedUrls(ctx context.Context, upReq *UploadResp, start, end int) (*S3PreSignedURLs, error) {
+	data := base.Json{
+		"bucket":          upReq.Data.Bucket,
+		"key":             upReq.Data.Key,
+		"partNumberEnd":   end,
+		"partNumberStart": start,
+		"uploadId":        upReq.Data.UploadId,
+		"StorageNode":     upReq.Data.StorageNode,
+	}
+	var s3PreSignedUrls S3PreSignedURLs
+	_, err := d.request(S3PreSignedUrls, http.MethodPost, func(req *resty.Request) {
+		req.SetBody(data).SetContext(ctx)
+	}, &s3PreSignedUrls)
+	if err != nil {
+		return nil, err
+	}
+	return &s3PreSignedUrls, nil
+}
+
+func (d *Pan123) completeS3(ctx context.Context, upReq *UploadResp) error {
+	data := base.Json{
+		"bucket":      upReq.Data.Bucket,
+		"key":         upReq.Data.Key,
+		"uploadId":    upReq.Data.UploadId,
+		"StorageNode": upReq.Data.StorageNode,
+	}
+	_, err := d.request(S3Complete, http.MethodPost, func(req *resty.Request) {
+		req.SetBody(data).SetContext(ctx)
+	}, nil)
+	return err
+}
+
+func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.FileStreamer, reader io.Reader, up driver.UpdateProgress) error {
+	chunkSize := int64(1024 * 1024 * 5)
+	// fetch s3 pre signed urls
+	chunkCount := int(math.Ceil(float64(file.GetSize()) / float64(chunkSize)))
+	// upload 10 chunks each batch
+	batchSize := 10
+	for i := 1; i <= chunkCount; i += batchSize {
+		if utils.IsCanceled(ctx) {
+			return ctx.Err()
+		}
+		start := i
+		end := i + batchSize
+		if end > chunkCount+1 {
+			end = chunkCount + 1
+		}
+		s3PreSignedUrls, err := d.getS3PreSignedUrls(ctx, upReq, start, end)
+		if err != nil {
+			return err
+		}
+		// upload each chunk
+		for j := start; j < end; j++ {
+			if utils.IsCanceled(ctx) {
+				return ctx.Err()
+			}
+			curSize := chunkSize
+			if j == chunkCount {
+				curSize = file.GetSize() - (int64(chunkCount)-1)*chunkSize
+			}
+			err = d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, j, end, io.LimitReader(reader, chunkSize), curSize, false)
+			if err != nil {
+				return err
+			}
+			up(j * 100 / chunkCount)
+		}
+	}
+	// complete s3 upload
+	return d.completeS3(ctx, upReq)
+}
+
+func (d *Pan123) uploadS3Chunk(ctx context.Context, upReq *UploadResp, s3PreSignedUrls *S3PreSignedURLs, cur, end int, reader io.Reader, curSize int64, retry bool) error {
+	uploadUrl := s3PreSignedUrls.Data.PreSignedUrls[strconv.Itoa(cur)]
+	if uploadUrl == "" {
+		return fmt.Errorf("upload url is empty, s3PreSignedUrls: %+v", s3PreSignedUrls)
+	}
+	req, err := http.NewRequest("PUT", uploadUrl, reader)
+	if err != nil {
+		return err
+	}
+	req = req.WithContext(ctx)
+	req.ContentLength = curSize
+	//req.Header.Set("Content-Length", strconv.FormatInt(curSize, 10))
+	res, err := base.HttpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusForbidden {
+		if retry {
+			return fmt.Errorf("upload s3 chunk %d failed, status code: %d", cur, res.StatusCode)
+		}
+		// refresh s3 pre signed urls
+		newS3PreSignedUrls, err := d.getS3PreSignedUrls(ctx, upReq, cur, end)
+		if err != nil {
+			return err
+		}
+		s3PreSignedUrls.Data.PreSignedUrls = newS3PreSignedUrls.Data.PreSignedUrls
+		// retry
+		return d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, cur, end, reader, curSize, true)
+	}
+	if res.StatusCode != http.StatusOK {
+		body, err := io.ReadAll(res.Body)
+		if err != nil {
+			return err
+		}
+		return fmt.Errorf("upload s3 chunk %d failed, status code: %d, body: %s", cur, res.StatusCode, body)
+	}
+	return nil
+}

drivers/123/util.go

@@ -14,9 +14,24 @@ import (
 
 // do others that not defined in Driver interface
 
+const (
+	API             = "https://www.123pan.com/b/api"
+	SignIn          = API + "/user/sign_in"
+	UserInfo        = API + "/user/info"
+	FileList        = API + "/file/list/new"
+	DownloadInfo    = API + "/file/download_info"
+	Mkdir           = API + "/file/upload_request"
+	Move            = API + "/file/mod_pid"
+	Rename          = API + "/file/rename"
+	Trash           = API + "/file/trash"
+	UploadRequest   = API + "/file/upload_request"
+	UploadComplete  = API + "/file/upload_complete"
+	S3PreSignedUrls = API + "/file/s3_repare_upload_parts_batch"
+	S3Complete      = API + "/file/s3_complete_multipart_upload"
+)
+
 func (d *Pan123) login() error {
 	var body base.Json
-	url := "https://www.123pan.com/a/api/user/sign_in"
 	if utils.IsEmailFormat(d.Username) {
 		body = base.Json{
 			"mail":     d.Username,
@@ -30,7 +45,7 @@ func (d *Pan123) login() error {
 		}
 	}
 	res, err := base.RestyClient.R().
-		SetBody(body).Post(url)
+		SetBody(body).Post(SignIn)
 	if err != nil {
 		return err
 	}
@@ -90,7 +105,7 @@ func (d *Pan123) getFiles(parentId string) ([]File, error) {
 			"trashed":        "false",
 			"Page":           strconv.Itoa(page),
 		}
-		_, err := d.request("https://www.123pan.com/api/file/list/new", http.MethodGet, func(req *resty.Request) {
+		_, err := d.request(FileList, http.MethodGet, func(req *resty.Request) {
 			req.SetQueryParams(query)
 		}, &resp)
 		if err != nil {

drivers/alias/util.go

@@ -51,7 +51,7 @@ func (d *Alias) getRootAndPath(path string) (string, string) {
 }
 
 func (d *Alias) get(ctx context.Context, path string, dst, sub string) (model.Obj, error) {
-	obj, err := fs.Get(ctx, stdpath.Join(dst, sub))
+	obj, err := fs.Get(ctx, stdpath.Join(dst, sub), &fs.GetArgs{NoLog: true})
 	if err != nil {
 		return nil, err
 	}
@@ -65,7 +65,7 @@ func (d *Alias) get(ctx context.Context, path string, dst, sub string) (model.Ob
 }
 
 func (d *Alias) list(ctx context.Context, dst, sub string) ([]model.Obj, error) {
-	objs, err := fs.List(ctx, stdpath.Join(dst, sub))
+	objs, err := fs.List(ctx, stdpath.Join(dst, sub), &fs.ListArgs{NoLog: true})
 	// the obj must implement the model.SetPath interface
 	// return objs, err
 	if err != nil {
@@ -83,11 +83,11 @@ func (d *Alias) list(ctx context.Context, dst, sub string) ([]model.Obj, error)
 
 func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs) (*model.Link, error) {
 	reqPath := stdpath.Join(dst, sub)
-	storage, err := fs.GetStorage(reqPath)
+	storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{})
 	if err != nil {
 		return nil, err
 	}
-	_, err = fs.Get(ctx, reqPath)
+	_, err = fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true})
 	if err != nil {
 		return nil, err
 	}

drivers/aliyundrive_open/driver.go

@@ -5,7 +5,7 @@ import (
 	"io"
 	"math"
 	"net/http"
-	"strings"
+	"time"
 
 	"github.com/alist-org/alist/v3/drivers/base"
 	"github.com/alist-org/alist/v3/internal/driver"
@@ -148,11 +148,7 @@ func (d *AliyundriveOpen) Put(ctx context.Context, dstDir model.Obj, stream mode
 	count := 1
 	if stream.GetSize() > DEFAULT {
 		count = int(math.Ceil(float64(stream.GetSize()) / float64(DEFAULT)))
-		partInfoList := make([]base.Json, 0, count)
-		for i := 1; i <= count; i++ {
-			partInfoList = append(partInfoList, base.Json{"part_number": i})
-		}
-		createData["part_info_list"] = partInfoList
+		createData["part_info_list"] = makePartInfos(count)
 	}
 	var createResp CreateResp
 	_, err := d.request("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) {
@@ -162,28 +158,26 @@ func (d *AliyundriveOpen) Put(ctx context.Context, dstDir model.Obj, stream mode
 		return err
 	}
 	// 2. upload
-	for i, partInfo := range createResp.PartInfoList {
+	preTime := time.Now()
+	for i := 1; i <= len(createResp.PartInfoList); i++ {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()
 		}
-		uploadUrl := partInfo.UploadUrl
-		if d.InternalUpload {
-			//Replace a known public Host with an internal Host
-			uploadUrl = strings.ReplaceAll(uploadUrl, "https://cn-beijing-data.aliyundrive.net/", "http://ccp-bj29-bj-1592982087.oss-cn-beijing-internal.aliyuncs.com/")
-		}
-		req, err := http.NewRequest("PUT", uploadUrl, io.LimitReader(stream, DEFAULT))
+		err = d.uploadPart(ctx, i, count, utils.NewMultiReadable(io.LimitReader(stream, DEFAULT)), &createResp, true)
 		if err != nil {
 			return err
 		}
-		req = req.WithContext(ctx)
-		res, err := base.HttpClient.Do(req)
-		if err != nil {
-			return err
-		}
-		res.Body.Close()
 		if count > 0 {
 			up(i * 100 / count)
 		}
+		// refresh upload url if 50 minutes passed
+		if time.Since(preTime) > 50*time.Minute {
+			createResp.PartInfoList, err = d.getUploadUrl(count, createResp.FileId, createResp.UploadId)
+			if err != nil {
+				return err
+			}
+			preTime = time.Now()
+		}
 	}
 	// 3. complete
 	_, err = d.request("/adrive/v1.0/openFile/complete", http.MethodPost, func(req *resty.Request) {

drivers/aliyundrive_open/meta.go

@@ -14,7 +14,7 @@ type Addition struct {
 	ClientID       string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"`
 	ClientSecret   string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"`
 	RemoveWay      string `json:"remove_way" required:"true" type:"select" options:"trash,delete"`
-	InternalUpload bool   `json:"internal_upload" help:"If you are using Aliyun ECS in Beijing, you can turn it on to boost the upload speed"`
+	InternalUpload bool   `json:"internal_upload" help:"If you are using Aliyun ECS is located in Beijing, you can turn it on to boost the upload speed"`
 	AccessToken    string
 }
 

drivers/aliyundrive_open/types.go

@@ -45,6 +45,14 @@ func fileToObj(f File) *model.ObjThumb {
 	}
 }
 
+type PartInfo struct {
+	Etag        interface{} `json:"etag"`
+	PartNumber  int         `json:"part_number"`
+	PartSize    interface{} `json:"part_size"`
+	UploadUrl   string      `json:"upload_url"`
+	ContentType string      `json:"content_type"`
+}
+
 type CreateResp struct {
 	//Type         string `json:"type"`
 	//ParentFileId string `json:"parent_file_id"`
@@ -56,12 +64,6 @@ type CreateResp struct {
 	//FileName     string `json:"file_name"`
 	UploadId string `json:"upload_id"`
 	//Location     string `json:"location"`
-	RapidUpload  bool `json:"rapid_upload"`
-	PartInfoList []struct {
-		Etag        interface{} `json:"etag"`
-		PartNumber  int         `json:"part_number"`
-		PartSize    interface{} `json:"part_size"`
-		UploadUrl   string      `json:"upload_url"`
-		ContentType string      `json:"content_type"`
-	} `json:"part_info_list"`
+	RapidUpload  bool       `json:"rapid_upload"`
+	PartInfoList []PartInfo `json:"part_info_list"`
 }

drivers/aliyundrive_open/util.go

@@ -1,12 +1,15 @@
 package aliyundrive_open
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/alist-org/alist/v3/drivers/base"
 	"github.com/alist-org/alist/v3/internal/op"
+	"github.com/alist-org/alist/v3/pkg/utils"
 	"github.com/go-resty/resty/v2"
 )
 
@@ -106,3 +109,59 @@ func (d *AliyundriveOpen) getFiles(fileId string) ([]File, error) {
 	}
 	return res, nil
 }
+
+func makePartInfos(size int) []base.Json {
+	partInfoList := make([]base.Json, size)
+	for i := 0; i < size; i++ {
+		partInfoList[i] = base.Json{"part_number": 1 + i}
+	}
+	return partInfoList
+}
+
+func (d *AliyundriveOpen) getUploadUrl(count int, fileId, uploadId string) ([]PartInfo, error) {
+	partInfoList := makePartInfos(count)
+	var resp CreateResp
+	_, err := d.request("/adrive/v1.0/openFile/getUploadUrl", http.MethodPost, func(req *resty.Request) {
+		req.SetBody(base.Json{
+			"drive_id":       d.DriveId,
+			"file_id":        fileId,
+			"part_info_list": partInfoList,
+			"upload_id":      uploadId,
+		}).SetResult(&resp)
+	})
+	return resp.PartInfoList, err
+}
+
+func (d *AliyundriveOpen) uploadPart(ctx context.Context, i, count int, reader *utils.MultiReadable, resp *CreateResp, retry bool) error {
+	partInfo := resp.PartInfoList[i-1]
+	uploadUrl := partInfo.UploadUrl
+	if d.InternalUpload {
+		uploadUrl = strings.ReplaceAll(uploadUrl, "https://cn-beijing-data.aliyundrive.net/", "http://ccp-bj29-bj-1592982087.oss-cn-beijing-internal.aliyuncs.com/")
+	}
+	req, err := http.NewRequest("PUT", uploadUrl, reader)
+	if err != nil {
+		return err
+	}
+	req = req.WithContext(ctx)
+	res, err := base.HttpClient.Do(req)
+	if err != nil {
+		if retry {
+			reader.Reset()
+			return d.uploadPart(ctx, i, count, reader, resp, false)
+		}
+		return err
+	}
+	res.Body.Close()
+	if retry && res.StatusCode == http.StatusForbidden {
+		resp.PartInfoList, err = d.getUploadUrl(count, resp.FileId, resp.UploadId)
+		if err != nil {
+			return err
+		}
+		reader.Reset()
+		return d.uploadPart(ctx, i, count, reader, resp, false)
+	}
+	if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusConflict {
+		return fmt.Errorf("upload status: %d", res.StatusCode)
+	}
+	return nil
+}

drivers/onedrive/util.go

@@ -193,6 +193,9 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil
 		req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()))
 		finish += byteSize
 		res, err := base.HttpClient.Do(req)
+		if err != nil {
+			return err
+		}
 		if res.StatusCode != 201 && res.StatusCode != 202 {
 			data, _ := io.ReadAll(res.Body)
 			res.Body.Close()

drivers/onedrive_app/util.go

@@ -184,6 +184,9 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model.
 		req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()))
 		finish += byteSize
 		res, err := base.HttpClient.Do(req)
+		if err != nil {
+			return err
+		}
 		if res.StatusCode != 201 && res.StatusCode != 202 {
 			data, _ := io.ReadAll(res.Body)
 			res.Body.Close()

drivers/quark/driver.go

@@ -56,8 +56,10 @@ func (d *Quark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
 		"fids": []string{file.GetID()},
 	}
 	var resp DownResp
+	ua := "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch"
 	_, err := d.request("/file/download", http.MethodPost, func(req *resty.Request) {
-		req.SetBody(data)
+		req.SetHeader("User-Agent", ua).
+			SetBody(data)
 	}, &resp)
 	if err != nil {
 		return nil, err
@@ -65,8 +67,9 @@ func (d *Quark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
 	return &model.Link{
 		URL: resp.Data[0].DownloadUrl,
 		Header: http.Header{
-			"Cookie":  []string{d.Cookie},
-			"Referer": []string{"https://pan.quark.cn"},
+			"Cookie":     []string{d.Cookie},
+			"Referer":    []string{"https://pan.quark.cn"},
+			"User-Agent": []string{ua},
 		},
 	}, nil
 }

drivers/s3/driver.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/url"
 	stdpath "path"
+	"strings"
 	"time"
 
 	"github.com/alist-org/alist/v3/internal/driver"
@@ -75,6 +76,9 @@ func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*mo
 	if d.CustomHost != "" {
 		err = req.Build()
 		link = req.HTTPRequest.URL.String()
+		if d.RemoveBucket {
+			link = strings.Replace(link, "/"+d.Bucket, "", 1)
+		}
 	} else {
 		link, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire))
 	}
@@ -128,6 +132,9 @@ func (d *S3) Remove(ctx context.Context, obj model.Obj) error {
 
 func (d *S3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
 	uploader := s3manager.NewUploader(d.Session)
+	if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize {
+		uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1)
+	}
 	key := getKey(stdpath.Join(dstDir.GetPath(), stream.GetName()), false)
 	log.Debugln("key:", key)
 	input := &s3manager.UploadInput{

drivers/s3/meta.go

@@ -17,6 +17,7 @@ type Addition struct {
 	Placeholder       string `json:"placeholder"`
 	ForcePathStyle    bool   `json:"force_path_style"`
 	ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"`
+	RemoveBucket      bool   `json:"remove_bucket" help:"Remove bucket name from path when using custom host."`
 }
 
 var config = driver.Config{

drivers/trainbit/util.go

@@ -113,10 +113,10 @@ func parseRawFileObject(rawObject []any) ([]model.Obj, error) {
 		}
 		isFolder := int64(object["ty"].(float64)) == 1
 		var name string
-		if isFolder {
-			name = object["name"].(string)
+		if object["ext"].(string) != "" {
+			name = strings.Join([]string{object["name"].(string), object["ext"].(string)}, ".")
 		} else {
-			name = object["name"].(string) + object["ext"].(string)
+			name = object["name"].(string)
 		}
 		modified, err := time.Parse("2006/01/02 15:04:05", object["modified"].(string))
 		if err != nil {

go.mod

@@ -1,6 +1,6 @@
 module github.com/alist-org/alist/v3
 
-go 1.19
+go 1.20
 
 require (
 	github.com/SheltonZhu/115driver v1.0.14
@@ -26,14 +26,14 @@ require (
 	github.com/pkg/sftp v1.13.5
 	github.com/pquerna/otp v1.4.0
 	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v1.6.1
+	github.com/spf13/cobra v1.7.0
 	github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca
 	github.com/u2takey/ffmpeg-go v0.4.1
-	github.com/upyun/go-sdk/v3 v3.0.3
+	github.com/upyun/go-sdk/v3 v3.0.4
 	github.com/winfsp/cgofuse v1.5.0
-	golang.org/x/crypto v0.7.0
-	golang.org/x/image v0.6.0
-	golang.org/x/net v0.8.0
+	golang.org/x/crypto v0.8.0
+	golang.org/x/image v0.7.0
+	golang.org/x/net v0.9.0
 	gorm.io/driver/mysql v1.4.7
 	gorm.io/driver/postgres v1.4.8
 	gorm.io/driver/sqlite v1.4.4
@@ -79,7 +79,7 @@ require (
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx/v5 v5.3.0 // indirect
@@ -104,8 +104,8 @@ require (
 	github.com/ugorji/go/codec v1.2.9 // indirect
 	go.etcd.io/bbolt v1.3.5 // indirect
 	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect

go.sum

@@ -129,8 +129,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI=
 github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
-github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
-github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
@@ -214,8 +214,8 @@ github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
-github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -244,8 +244,8 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6
 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
 github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU=
 github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
-github.com/upyun/go-sdk/v3 v3.0.3 h1:2wUkNk2fyJReMYHMvJyav050D83rYwSjN7mEPR0Pp8Q=
-github.com/upyun/go-sdk/v3 v3.0.3/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E=
+github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q=
+github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E=
 github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
 github.com/winfsp/cgofuse v1.5.0 h1:MsBP7Mi/LiJf/7/F3O/7HjjR009ds6KCdqXzKpZSWxI=
 github.com/winfsp/cgofuse v1.5.0/go.mod h1:h3awhoUOcn2VYVKCwDaYxSLlZwnyK+A8KaDoLUp2lbU=
@@ -265,11 +265,11 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
-golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
+golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.6.0 h1:bR8b5okrPI3g/gyZakLZHeWxAR8Dn5CyxXv1hLH5g/4=
-golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0=
+golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw=
+golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -281,8 +281,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -305,13 +305,13 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
+golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -319,8 +319,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y=

internal/fs/fs.go

@@ -13,19 +13,32 @@ import (
 // So, the purpose of this package is to convert mount path to actual path
 // then pass the actual path to the op package
 
-func List(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error) {
-	res, err := list(ctx, path, refresh...)
+type ListArgs struct {
+	Refresh bool
+	NoLog   bool
+}
+
+func List(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) {
+	res, err := list(ctx, path, args)
 	if err != nil {
-		log.Errorf("failed list %s: %+v", path, err)
+		if !args.NoLog {
+			log.Errorf("failed list %s: %+v", path, err)
+		}
 		return nil, err
 	}
 	return res, nil
 }
 
-func Get(ctx context.Context, path string) (model.Obj, error) {
+type GetArgs struct {
+	NoLog bool
+}
+
+func Get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) {
 	res, err := get(ctx, path)
 	if err != nil {
-		log.Errorf("failed get %s: %+v", path, err)
+		if !args.NoLog {
+			log.Errorf("failed get %s: %+v", path, err)
+		}
 		return nil, err
 	}
 	return res, nil
@@ -96,7 +109,10 @@ func PutAsTask(dstDirPath string, file *model.FileStream) error {
 	return err
 }
 
-func GetStorage(path string) (driver.Driver, error) {
+type GetStoragesArgs struct {
+}
+
+func GetStorage(path string, args *GetStoragesArgs) (driver.Driver, error) {
 	storageDriver, _, err := op.GetStorageAndActualPath(path)
 	if err != nil {
 		return nil, err

internal/fs/list.go

@@ -11,7 +11,7 @@ import (
 )
 
 // List files
-func list(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error) {
+func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) {
 	meta := ctx.Value("meta").(*model.Meta)
 	user := ctx.Value("user").(*model.User)
 	virtualFiles := op.GetStorageVirtualFilesByPath(path)
@@ -24,9 +24,11 @@ func list(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error
 	if storage != nil {
 		_objs, err = op.List(ctx, storage, actualPath, model.ListArgs{
 			ReqPath: path,
-		}, refresh...)
+		}, args.Refresh)
 		if err != nil {
-			log.Errorf("%+v", err)
+			if !args.NoLog {
+				log.Errorf("fs/list: %+v", err)
+			}
 			if len(virtualFiles) == 0 {
 				return nil, errors.WithMessage(err, "failed get objs")
 			}

internal/fs/walk.go

@@ -28,7 +28,7 @@ func WalkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn
 	}
 	meta, _ := op.GetNearestMeta(name)
 	// Read directory names.
-	objs, err := List(context.WithValue(ctx, "meta", meta), name)
+	objs, err := List(context.WithValue(ctx, "meta", meta), name, &ListArgs{})
 	if err != nil {
 		return walkFnErr
 	}

internal/search/build.go

@@ -141,7 +141,7 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth
 			})
 			return nil
 		}
-		fi, err = fs.Get(ctx, indexPath)
+		fi, err = fs.Get(ctx, indexPath, &fs.GetArgs{})
 		if err != nil {
 			return err
 		}

pkg/gowebdav/client.go

@@ -343,9 +343,9 @@ func (c *Client) Read(path string) ([]byte, error) {
 }
 
 func (c *Client) Link(path string) (string, http.Header, error) {
-	method := "HEAD"
-	url := PathEscape(Join(c.root, path))
-	r, err := http.NewRequest(method, url, nil)
+	method := "GET"
+	u := PathEscape(Join(c.root, path))
+	r, err := http.NewRequest(method, u, nil)
 
 	if err != nil {
 		return "", nil, newPathErrorErr("Link", path, err)
@@ -366,31 +366,6 @@ func (c *Client) Link(path string) (string, http.Header, error) {
 	if c.interceptor != nil {
 		c.interceptor(method, r)
 	}
-
-	rs, err := c.c.Do(r)
-	if err != nil {
-		return "", nil, newPathErrorErr("Link", path, err)
-	}
-
-	if rs.StatusCode == 401 {
-		wwwAuthenticateHeader := strings.ToLower(rs.Header.Get("Www-Authenticate"))
-		if strings.Contains(wwwAuthenticateHeader, "digest") {
-			c.authMutex.Lock()
-			c.auth = &DigestAuth{auth.User(), auth.Pass(), digestParts(rs)}
-			c.auth.Authorize(r, method, path)
-			c.authMutex.Unlock()
-		} else if strings.Contains(wwwAuthenticateHeader, "basic") {
-			c.authMutex.Lock()
-			c.auth = &BasicAuth{auth.User(), auth.Pass()}
-			c.auth.Authorize(r, method, path)
-			c.authMutex.Unlock()
-		} else {
-			return "", nil, newPathError("Authorize", c.root, rs.StatusCode)
-		}
-	} else if rs.StatusCode > 400 {
-		return "", nil, newPathError("Authorize", path, rs.StatusCode)
-	}
-
 	return r.URL.String(), r.Header, nil
 }
 

pkg/task/task.go

@@ -3,6 +3,7 @@ package task
 
 import (
 	"context"
+	"runtime"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -63,11 +64,17 @@ func (t Task[K]) GetErrMsg() string {
 	return t.Error.Error()
 }
 
+func getCurrentGoroutineStack() string {
+	buf := make([]byte, 1<<16)
+	n := runtime.Stack(buf, false)
+	return string(buf[:n])
+}
+
 func (t *Task[K]) run() {
 	t.state = RUNNING
 	defer func() {
 		if err := recover(); err != nil {
-			log.Errorf("error [%+v] while run task [%s]", err, t.Name)
+			log.Errorf("error [%s] while run task [%s],stack trace:\n%s", err, t.Name, getCurrentGoroutineStack())
 			t.Error = errors.Errorf("panic: %+v", err)
 			t.state = ERRORED
 		}

pkg/utils/io.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"bytes"
 	"context"
 	"io"
 )
@@ -91,3 +92,46 @@ func NewReadCloser(reader io.Reader, close CloseFunc) io.ReadCloser {
 func NewLimitReadCloser(reader io.Reader, close CloseFunc, limit int64) io.ReadCloser {
 	return NewReadCloser(io.LimitReader(reader, limit), close)
 }
+
+type MultiReadable struct {
+	originReader io.Reader
+	reader       io.Reader
+	cache        *bytes.Buffer
+}
+
+func NewMultiReadable(reader io.Reader) *MultiReadable {
+	return &MultiReadable{
+		originReader: reader,
+		reader:       reader,
+	}
+}
+
+func (mr *MultiReadable) Read(p []byte) (int, error) {
+	n, err := mr.reader.Read(p)
+	if _, ok := mr.reader.(io.Seeker); !ok && n > 0 {
+		if mr.cache == nil {
+			mr.cache = &bytes.Buffer{}
+		}
+		mr.cache.Write(p[:n])
+	}
+	return n, err
+}
+
+func (mr *MultiReadable) Reset() error {
+	if seeker, ok := mr.reader.(io.Seeker); ok {
+		_, err := seeker.Seek(0, io.SeekStart)
+		return err
+	}
+	if mr.cache != nil && mr.cache.Len() > 0 {
+		mr.reader = io.MultiReader(mr.cache, mr.reader)
+		mr.cache = nil
+	}
+	return nil
+}
+
+func (mr *MultiReadable) Close() error {
+	if closer, ok := mr.originReader.(io.Closer); ok {
+		return closer.Close()
+	}
+	return nil
+}

server/common/proxy.go

@@ -16,7 +16,15 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-var HttpClient = &http.Client{}
+var HttpClient = &http.Client{
+	CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		if len(via) >= 10 {
+			return errors.New("stopped after 10 redirects")
+		}
+		req.Header.Del("Referer")
+		return nil
+	},
+}
 
 func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error {
 	// read data with native

server/handles/down.go

@@ -21,7 +21,7 @@ import (
 func Down(c *gin.Context) {
 	rawPath := c.MustGet("path").(string)
 	filename := stdpath.Base(rawPath)
-	storage, err := fs.GetStorage(rawPath)
+	storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
@@ -65,7 +65,7 @@ func Down(c *gin.Context) {
 func Proxy(c *gin.Context) {
 	rawPath := c.MustGet("path").(string)
 	filename := stdpath.Base(rawPath)
-	storage, err := fs.GetStorage(rawPath)
+	storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return

server/handles/fsmanage.go

@@ -133,7 +133,7 @@ func FsRecursiveMove(c *gin.Context) {
 	}
 	c.Set("meta", meta)
 
-	rootFiles, err := fs.List(c, srcDir, false)
+	rootFiles, err := fs.List(c, srcDir, &fs.ListArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
@@ -154,7 +154,7 @@ func FsRecursiveMove(c *gin.Context) {
 		if movingFile.IsDir() {
 			// directory, recursive move
 			subFilePath := movingFilePath
-			subFiles, err := fs.List(c, subFilePath, true)
+			subFiles, err := fs.List(c, subFilePath, &fs.ListArgs{Refresh: true})
 			if err != nil {
 				common.ErrorResp(c, err, 500)
 				return
@@ -293,7 +293,7 @@ func FsRegexRename(c *gin.Context) {
 		return
 	}
 
-	files, err := fs.List(c, reqPath, false)
+	files, err := fs.List(c, reqPath, &fs.ListArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
@@ -351,6 +351,105 @@ func FsRemove(c *gin.Context) {
 	common.SuccessResp(c)
 }
 
+type RemoveEmptyDirectoryReq struct {
+	SrcDir string `json:"src_dir"`
+}
+
+func FsRemoveEmptyDirectory(c *gin.Context) {
+	var req RemoveEmptyDirectoryReq
+	if err := c.ShouldBind(&req); err != nil {
+		common.ErrorResp(c, err, 400)
+		return
+	}
+
+	user := c.MustGet("user").(*model.User)
+	if !user.CanRemove() {
+		common.ErrorResp(c, errs.PermissionDenied, 403)
+		return
+	}
+	srcDir, err := user.JoinPath(req.SrcDir)
+	if err != nil {
+		common.ErrorResp(c, err, 403)
+		return
+	}
+
+	meta, err := op.GetNearestMeta(srcDir)
+	if err != nil {
+		if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
+			common.ErrorResp(c, err, 500, true)
+			return
+		}
+	}
+	c.Set("meta", meta)
+
+	rootFiles, err := fs.List(c, srcDir, &fs.ListArgs{})
+	if err != nil {
+		common.ErrorResp(c, err, 500)
+		return
+	}
+
+	// record the file path
+	filePathMap := make(map[model.Obj]string)
+	// record the parent file
+	fileParentMap := make(map[model.Obj]model.Obj)
+	// removing files
+	removingFiles := generic.NewQueue[model.Obj]()
+	// removed files
+	removedFiles := make(map[string]bool)
+	for _, file := range rootFiles {
+		if !file.IsDir() {
+			continue
+		}
+		removingFiles.Push(file)
+		filePathMap[file] = srcDir
+	}
+
+	for !removingFiles.IsEmpty() {
+
+		removingFile := removingFiles.Pop()
+		removingFilePath := fmt.Sprintf("%s/%s", filePathMap[removingFile], removingFile.GetName())
+
+		if removedFiles[removingFilePath] {
+			continue
+		}
+
+		subFiles, err := fs.List(c, removingFilePath, &fs.ListArgs{Refresh: true})
+		if err != nil {
+			common.ErrorResp(c, err, 500)
+			return
+		}
+
+		if len(subFiles) == 0 {
+			// remove empty directory
+			err = fs.Remove(c, removingFilePath)
+			removedFiles[removingFilePath] = true
+			if err != nil {
+				common.ErrorResp(c, err, 500)
+				return
+			}
+			// recheck parent folder
+			parentFile, exist := fileParentMap[removingFile]
+			if exist {
+				removingFiles.Push(parentFile)
+			}
+
+		} else {
+			// recursive remove
+			for _, subFile := range subFiles {
+				if !subFile.IsDir() {
+					continue
+				}
+				removingFiles.Push(subFile)
+				filePathMap[subFile] = removingFilePath
+				fileParentMap[subFile] = removingFile
+			}
+		}
+
+	}
+
+	common.SuccessResp(c)
+}
+
 // Link return real link, just for proxy program, it may contain cookie, so just allowed for admin
 func Link(c *gin.Context) {
 	var req MkdirOrLinkReq
@@ -362,7 +461,7 @@ func Link(c *gin.Context) {
 	//rawPath := stdpath.Join(user.BasePath, req.Path)
 	// why need not join base_path? because it's always the full path
 	rawPath := req.Path
-	storage, err := fs.GetStorage(rawPath)
+	storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return

server/handles/fsread.go

@@ -6,10 +6,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/alist-org/alist/v3/internal/conf"
 	"github.com/alist-org/alist/v3/internal/errs"
 	"github.com/alist-org/alist/v3/internal/fs"
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/internal/op"
+	"github.com/alist-org/alist/v3/internal/setting"
 	"github.com/alist-org/alist/v3/internal/sign"
 	"github.com/alist-org/alist/v3/pkg/utils"
 	"github.com/alist-org/alist/v3/server/common"
@@ -77,14 +79,14 @@ func FsList(c *gin.Context) {
 		common.ErrorStrResp(c, "Refresh without permission", 403)
 		return
 	}
-	objs, err := fs.List(c, reqPath, req.Refresh)
+	objs, err := fs.List(c, reqPath, &fs.ListArgs{Refresh: req.Refresh})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
 	}
 	total, objs := pagination(objs, &req.PageReq)
 	provider := "unknown"
-	storage, err := fs.GetStorage(reqPath)
+	storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{})
 	if err == nil {
 		provider = storage.GetStorage().Driver
 	}
@@ -130,7 +132,7 @@ func FsDirs(c *gin.Context) {
 		common.ErrorStrResp(c, "password is incorrect or you have no permission", 403)
 		return
 	}
-	objs, err := fs.List(c, reqPath)
+	objs, err := fs.List(c, reqPath, &fs.ListArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
@@ -245,14 +247,14 @@ func FsGet(c *gin.Context) {
 		common.ErrorStrResp(c, "password is incorrect or you have no permission", 403)
 		return
 	}
-	obj, err := fs.Get(c, reqPath)
+	obj, err := fs.Get(c, reqPath, &fs.GetArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 500)
 		return
 	}
 	var rawURL string
 
-	storage, err := fs.GetStorage(reqPath)
+	storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{})
 	provider := "unknown"
 	if err == nil {
 		provider = storage.Config().Name
@@ -264,7 +266,7 @@ func FsGet(c *gin.Context) {
 		}
 		if storage.Config().MustProxy() || storage.GetStorage().WebProxy {
 			query := ""
-			if isEncrypt(meta, reqPath) {
+			if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) {
 				query = "?sign=" + sign.Sign(reqPath)
 			}
 			if storage.GetStorage().DownProxyUrl != "" {
@@ -295,7 +297,7 @@ func FsGet(c *gin.Context) {
 	}
 	var related []model.Obj
 	parentPath := stdpath.Dir(reqPath)
-	sameLevelFiles, err := fs.List(c, parentPath)
+	sameLevelFiles, err := fs.List(c, parentPath, &fs.ListArgs{})
 	if err == nil {
 		related = filterRelated(sameLevelFiles, obj)
 	}

server/handles/fsup.go

@@ -69,7 +69,7 @@ func FsForm(c *gin.Context) {
 		common.ErrorResp(c, err, 403)
 		return
 	}
-	storage, err := fs.GetStorage(path)
+	storage, err := fs.GetStorage(path, &fs.GetStoragesArgs{})
 	if err != nil {
 		common.ErrorResp(c, err, 400)
 		return

server/router.go

@@ -131,6 +131,7 @@ func _fs(g *gin.RouterGroup) {
 	g.POST("/recursive_move", handles.FsRecursiveMove)
 	g.POST("/copy", handles.FsCopy)
 	g.POST("/remove", handles.FsRemove)
+	g.POST("/remove_empty_directory", handles.FsRemoveEmptyDirectory)
 	g.PUT("/put", middlewares.FsUp, handles.FsStream)
 	g.PUT("/form", middlewares.FsUp, handles.FsForm)
 	g.POST("/link", middlewares.AuthAdmin, handles.Link)

server/webdav/file.go

@@ -84,7 +84,7 @@ func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn
 	}
 	meta, _ := op.GetNearestMeta(name)
 	// Read directory names.
-	objs, err := fs.List(context.WithValue(ctx, "meta", meta), name)
+	objs, err := fs.List(context.WithValue(ctx, "meta", meta), name, &fs.ListArgs{})
 	//f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0)
 	//if err != nil {
 	//	return walkFn(name, info, err)

server/webdav/webdav.go

@@ -188,7 +188,7 @@ func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status
 		return 403, err
 	}
 	allow := "OPTIONS, LOCK, PUT, MKCOL"
-	if fi, err := fs.Get(ctx, reqPath); err == nil {
+	if fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err == nil {
 		if fi.IsDir() {
 			allow = "OPTIONS, LOCK, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND"
 		} else {
@@ -215,7 +215,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta
 	if err != nil {
 		return 403, err
 	}
-	fi, err := fs.Get(ctx, reqPath)
+	fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
 	if err != nil {
 		return http.StatusNotFound, err
 	}
@@ -228,7 +228,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta
 	}
 	w.Header().Set("ETag", etag)
 	// Let ServeContent determine the Content-Type header.
-	storage, _ := fs.GetStorage(reqPath)
+	storage, _ := fs.GetStorage(reqPath, &fs.GetStoragesArgs{})
 	downProxyUrl := storage.GetStorage().DownProxyUrl
 	if storage.GetStorage().WebdavNative() || (storage.GetStorage().WebdavProxy() && downProxyUrl == "") {
 		link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{Header: r.Header})
@@ -278,7 +278,7 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i
 	// "godoc os RemoveAll" says that "If the path does not exist, RemoveAll
 	// returns nil (no error)." WebDAV semantics are that it should return a
 	// "404 Not Found". We therefore have to Stat before we RemoveAll.
-	if _, err := fs.Get(ctx, reqPath); err != nil {
+	if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil {
 		if errs.IsObjectNotFound(err) {
 			return http.StatusNotFound, err
 		}
@@ -331,7 +331,7 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int,
 	if err != nil {
 		return http.StatusMethodNotAllowed, err
 	}
-	fi, err := fs.Get(ctx, reqPath)
+	fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
 	if err != nil {
 		fi = &obj
 	}
@@ -591,7 +591,7 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status
 	if err != nil {
 		return 403, err
 	}
-	fi, err := fs.Get(ctx, reqPath)
+	fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{})
 	if err != nil {
 		if errs.IsObjectNotFound(err) {
 			return http.StatusNotFound, err
@@ -670,7 +670,7 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu
 	if err != nil {
 		return 403, err
 	}
-	if _, err := fs.Get(ctx, reqPath); err != nil {
+	if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil {
 		if errs.IsObjectNotFound(err) {
 			return http.StatusNotFound, err
 		}