fix(123): adapt new upload method (close #4141)

* 增加清理空文件夹API

* 修复嵌套文件夹删除Bug

 Author:    varg247 <varg247@gmail.com>

---------

Co-authored-by: varg247 <varg247@qq.com>

.github/workflows/auto_lang.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         platform: [ ubuntu-latest ]
-        go-version: [ 1.19 ]
+        go-version: [ '1.20' ]
     name: auto generate lang.json
     runs-on: ${{ matrix.platform }}
     steps:

.github/workflows/build.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest]
-        go-version: [1.19]
+        go-version: [ '1.20' ]
     name: Build
     runs-on: ${{ matrix.platform }}
     steps:

.github/workflows/release.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       matrix:
         platform: [ ubuntu-latest ]
-        go-version: [ 1.19 ]
+        go-version: [ '1.20' ]
     name: Release
     runs-on: ${{ matrix.platform }}
     steps:

Dockerfile

@@ -1,11 +1,11 @@
-FROM alpine:edge as builder
+FROM alpine:3.17 as builder
 LABEL stage=go-builder
 WORKDIR /app/
 COPY ./ ./
 RUN apk add --no-cache bash curl gcc git go musl-dev; \
     bash build.sh release docker
 
-FROM alpine:edge
+FROM alpine:3.17
 LABEL MAINTAINER="i@nn.ci"
 VOLUME /opt/alist/data/
 WORKDIR /opt/alist/

drivers/123/driver.go

@@ -220,32 +220,37 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
 		"type":         0,
 	}
 	var resp UploadResp
-	_, err := d.request(UploadRequest, http.MethodPost, func(req *resty.Request) {
+	res, err := d.request(UploadRequest, http.MethodPost, func(req *resty.Request) {
 		req.SetBody(data).SetContext(ctx)
 	}, &resp)
 	if err != nil {
 		return err
 	}
+	log.Debugln("upload request res: ", string(res))
 	if resp.Data.Reuse || resp.Data.Key == "" {
 		return nil
 	}
-	cfg := &aws.Config{
-		Credentials:      credentials.NewStaticCredentials(resp.Data.AccessKeyId, resp.Data.SecretAccessKey, resp.Data.SessionToken),
-		Region:           aws.String("123pan"),
-		Endpoint:         aws.String("file.123pan.com"),
-		S3ForcePathStyle: aws.Bool(true),
+	if resp.Data.AccessKeyId == "" || resp.Data.SecretAccessKey == "" || resp.Data.SessionToken == "" {
+		err = d.newUpload(ctx, &resp, stream, uploadFile, up)
+	} else {
+		cfg := &aws.Config{
+			Credentials:      credentials.NewStaticCredentials(resp.Data.AccessKeyId, resp.Data.SecretAccessKey, resp.Data.SessionToken),
+			Region:           aws.String("123pan"),
+			Endpoint:         aws.String(resp.Data.EndPoint),
+			S3ForcePathStyle: aws.Bool(true),
+		}
+		s, err := session.NewSession(cfg)
+		if err != nil {
+			return err
+		}
+		uploader := s3manager.NewUploader(s)
+		input := &s3manager.UploadInput{
+			Bucket: &resp.Data.Bucket,
+			Key:    &resp.Data.Key,
+			Body:   uploadFile,
+		}
+		_, err = uploader.UploadWithContext(ctx, input)
 	}
-	s, err := session.NewSession(cfg)
-	if err != nil {
-		return err
-	}
-	uploader := s3manager.NewUploader(s)
-	input := &s3manager.UploadInput{
-		Bucket: &resp.Data.Bucket,
-		Key:    &resp.Data.Key,
-		Body:   uploadFile,
-	}
-	_, err = uploader.UploadWithContext(ctx, input)
 	if err != nil {
 		return err
 	}

drivers/123/types.go

@@ -74,5 +74,14 @@ type UploadResp struct {
 		SessionToken    string `json:"SessionToken"`
 		FileId          int64  `json:"FileId"`
 		Reuse           bool   `json:"Reuse"`
+		EndPoint        string `json:"EndPoint"`
+		StorageNode     string `json:"StorageNode"`
+		UploadId        string `json:"UploadId"`
+	} `json:"data"`
+}
+
+type S3PreSignedURLs struct {
+	Data struct {
+		PreSignedUrls map[string]string `json:"presignedUrls"`
 	} `json:"data"`
 }

drivers/123/upload.go

@@ -0,0 +1,127 @@
+package _123
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"strconv"
+
+	"github.com/alist-org/alist/v3/drivers/base"
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/go-resty/resty/v2"
+)
+
+func (d *Pan123) getS3PreSignedUrls(ctx context.Context, upReq *UploadResp, start, end int) (*S3PreSignedURLs, error) {
+	data := base.Json{
+		"bucket":          upReq.Data.Bucket,
+		"key":             upReq.Data.Key,
+		"partNumberEnd":   end,
+		"partNumberStart": start,
+		"uploadId":        upReq.Data.UploadId,
+		"StorageNode":     upReq.Data.StorageNode,
+	}
+	var s3PreSignedUrls S3PreSignedURLs
+	_, err := d.request(S3PreSignedUrls, http.MethodPost, func(req *resty.Request) {
+		req.SetBody(data).SetContext(ctx)
+	}, &s3PreSignedUrls)
+	if err != nil {
+		return nil, err
+	}
+	return &s3PreSignedUrls, nil
+}
+
+func (d *Pan123) completeS3(ctx context.Context, upReq *UploadResp) error {
+	data := base.Json{
+		"bucket":      upReq.Data.Bucket,
+		"key":         upReq.Data.Key,
+		"uploadId":    upReq.Data.UploadId,
+		"StorageNode": upReq.Data.StorageNode,
+	}
+	_, err := d.request(S3Complete, http.MethodPost, func(req *resty.Request) {
+		req.SetBody(data).SetContext(ctx)
+	}, nil)
+	return err
+}
+
+func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.FileStreamer, reader io.Reader, up driver.UpdateProgress) error {
+	chunkSize := int64(1024 * 1024 * 5)
+	// fetch s3 pre signed urls
+	chunkCount := int(math.Ceil(float64(file.GetSize()) / float64(chunkSize)))
+	// upload 10 chunks each batch
+	batchSize := 10
+	for i := 1; i <= chunkCount; i += batchSize {
+		if utils.IsCanceled(ctx) {
+			return ctx.Err()
+		}
+		start := i
+		end := i + batchSize
+		if end > chunkCount+1 {
+			end = chunkCount + 1
+		}
+		s3PreSignedUrls, err := d.getS3PreSignedUrls(ctx, upReq, start, end)
+		if err != nil {
+			return err
+		}
+		// upload each chunk
+		for j := start; j < end; j++ {
+			if utils.IsCanceled(ctx) {
+				return ctx.Err()
+			}
+			curSize := chunkSize
+			if j == chunkCount {
+				curSize = file.GetSize() - (int64(chunkCount)-1)*chunkSize
+			}
+			err = d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, j, end, io.LimitReader(reader, chunkSize), curSize, false)
+			if err != nil {
+				return err
+			}
+			up(j * 100 / chunkCount)
+		}
+	}
+	// complete s3 upload
+	return d.completeS3(ctx, upReq)
+}
+
+func (d *Pan123) uploadS3Chunk(ctx context.Context, upReq *UploadResp, s3PreSignedUrls *S3PreSignedURLs, cur, end int, reader io.Reader, curSize int64, retry bool) error {
+	uploadUrl := s3PreSignedUrls.Data.PreSignedUrls[strconv.Itoa(cur)]
+	if uploadUrl == "" {
+		return fmt.Errorf("upload url is empty, s3PreSignedUrls: %+v", s3PreSignedUrls)
+	}
+	req, err := http.NewRequest("PUT", uploadUrl, reader)
+	if err != nil {
+		return err
+	}
+	req = req.WithContext(ctx)
+	req.ContentLength = curSize
+	//req.Header.Set("Content-Length", strconv.FormatInt(curSize, 10))
+	res, err := base.HttpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusForbidden {
+		if retry {
+			return fmt.Errorf("upload s3 chunk %d failed, status code: %d", cur, res.StatusCode)
+		}
+		// refresh s3 pre signed urls
+		newS3PreSignedUrls, err := d.getS3PreSignedUrls(ctx, upReq, cur, end)
+		if err != nil {
+			return err
+		}
+		s3PreSignedUrls.Data.PreSignedUrls = newS3PreSignedUrls.Data.PreSignedUrls
+		// retry
+		return d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, cur, end, reader, curSize, true)
+	}
+	if res.StatusCode != http.StatusOK {
+		body, err := io.ReadAll(res.Body)
+		if err != nil {
+			return err
+		}
+		return fmt.Errorf("upload s3 chunk %d failed, status code: %d, body: %s", cur, res.StatusCode, body)
+	}
+	return nil
+}

drivers/123/util.go

@@ -15,17 +15,19 @@ import (
 // do others that not defined in Driver interface
 
 const (
-	API            = "https://www.123pan.com/b/api"
-	SignIn         = API + "/user/sign_in"
-	UserInfo       = API + "/user/info"
-	FileList       = API + "/file/list/new"
-	DownloadInfo   = API + "/file/download_info"
-	Mkdir          = API + "/file/upload_request"
-	Move           = API + "/file/mod_pid"
-	Rename         = API + "/file/rename"
-	Trash          = API + "/file/trash"
-	UploadRequest  = API + "/file/upload_request"
-	UploadComplete = API + "/file/upload_complete"
+	API             = "https://www.123pan.com/b/api"
+	SignIn          = API + "/user/sign_in"
+	UserInfo        = API + "/user/info"
+	FileList        = API + "/file/list/new"
+	DownloadInfo    = API + "/file/download_info"
+	Mkdir           = API + "/file/upload_request"
+	Move            = API + "/file/mod_pid"
+	Rename          = API + "/file/rename"
+	Trash           = API + "/file/trash"
+	UploadRequest   = API + "/file/upload_request"
+	UploadComplete  = API + "/file/upload_complete"
+	S3PreSignedUrls = API + "/file/s3_repare_upload_parts_batch"
+	S3Complete      = API + "/file/s3_complete_multipart_upload"
 )
 
 func (d *Pan123) login() error {

drivers/alias/util.go

@@ -83,7 +83,7 @@ func (d *Alias) list(ctx context.Context, dst, sub string) ([]model.Obj, error)
 
 func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs) (*model.Link, error) {
 	reqPath := stdpath.Join(dst, sub)
-	storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{NoLog: true})
+	storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{})
 	if err != nil {
 		return nil, err
 	}

drivers/onedrive/util.go

@@ -193,6 +193,9 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil
 		req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()))
 		finish += byteSize
 		res, err := base.HttpClient.Do(req)
+		if err != nil {
+			return err
+		}
 		if res.StatusCode != 201 && res.StatusCode != 202 {
 			data, _ := io.ReadAll(res.Body)
 			res.Body.Close()

drivers/onedrive_app/util.go

@@ -184,6 +184,9 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model.
 		req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()))
 		finish += byteSize
 		res, err := base.HttpClient.Do(req)
+		if err != nil {
+			return err
+		}
 		if res.StatusCode != 201 && res.StatusCode != 202 {
 			data, _ := io.ReadAll(res.Body)
 			res.Body.Close()

drivers/quark/driver.go

@@ -56,8 +56,10 @@ func (d *Quark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
 		"fids": []string{file.GetID()},
 	}
 	var resp DownResp
+	ua := "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch"
 	_, err := d.request("/file/download", http.MethodPost, func(req *resty.Request) {
-		req.SetBody(data)
+		req.SetHeader("User-Agent", ua).
+			SetBody(data)
 	}, &resp)
 	if err != nil {
 		return nil, err
@@ -65,8 +67,9 @@ func (d *Quark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (
 	return &model.Link{
 		URL: resp.Data[0].DownloadUrl,
 		Header: http.Header{
-			"Cookie":  []string{d.Cookie},
-			"Referer": []string{"https://pan.quark.cn"},
+			"Cookie":     []string{d.Cookie},
+			"Referer":    []string{"https://pan.quark.cn"},
+			"User-Agent": []string{ua},
 		},
 	}, nil
 }

drivers/s3/driver.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/url"
 	stdpath "path"
+	"strings"
 	"time"
 
 	"github.com/alist-org/alist/v3/internal/driver"
@@ -75,6 +76,9 @@ func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*mo
 	if d.CustomHost != "" {
 		err = req.Build()
 		link = req.HTTPRequest.URL.String()
+		if d.RemoveBucket {
+			link = strings.Replace(link, "/"+d.Bucket, "", 1)
+		}
 	} else {
 		link, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire))
 	}

drivers/s3/meta.go

@@ -17,6 +17,7 @@ type Addition struct {
 	Placeholder       string `json:"placeholder"`
 	ForcePathStyle    bool   `json:"force_path_style"`
 	ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"`
+	RemoveBucket      bool   `json:"remove_bucket" help:"Remove bucket name from path when using custom host."`
 }
 
 var config = driver.Config{

go.mod

@@ -1,6 +1,6 @@
 module github.com/alist-org/alist/v3
 
-go 1.19
+go 1.20
 
 require (
 	github.com/SheltonZhu/115driver v1.0.14
@@ -31,9 +31,9 @@ require (
 	github.com/u2takey/ffmpeg-go v0.4.1
 	github.com/upyun/go-sdk/v3 v3.0.4
 	github.com/winfsp/cgofuse v1.5.0
-	golang.org/x/crypto v0.7.0
-	golang.org/x/image v0.6.0
-	golang.org/x/net v0.8.0
+	golang.org/x/crypto v0.8.0
+	golang.org/x/image v0.7.0
+	golang.org/x/net v0.9.0
 	gorm.io/driver/mysql v1.4.7
 	gorm.io/driver/postgres v1.4.8
 	gorm.io/driver/sqlite v1.4.4
@@ -104,8 +104,8 @@ require (
 	github.com/ugorji/go/codec v1.2.9 // indirect
 	go.etcd.io/bbolt v1.3.5 // indirect
 	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect

go.sum

@@ -265,11 +265,11 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
-golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
+golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.6.0 h1:bR8b5okrPI3g/gyZakLZHeWxAR8Dn5CyxXv1hLH5g/4=
-golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0=
+golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw=
+golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -281,8 +281,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -305,13 +305,13 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
+golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -319,8 +319,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y=

internal/fs/fs.go

@@ -21,7 +21,9 @@ type ListArgs struct {
 func List(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) {
 	res, err := list(ctx, path, args)
 	if err != nil {
-		log.Errorf("failed list %s: %+v", path, err)
+		if !args.NoLog {
+			log.Errorf("failed list %s: %+v", path, err)
+		}
 		return nil, err
 	}
 	return res, nil
@@ -33,8 +35,10 @@ type GetArgs struct {
 
 func Get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) {
 	res, err := get(ctx, path)
-	if err != nil && !args.NoLog {
-		log.Errorf("failed get %s: %+v", path, err)
+	if err != nil {
+		if !args.NoLog {
+			log.Errorf("failed get %s: %+v", path, err)
+		}
 		return nil, err
 	}
 	return res, nil
@@ -106,12 +110,11 @@ func PutAsTask(dstDirPath string, file *model.FileStream) error {
 }
 
 type GetStoragesArgs struct {
-	NoLog bool
 }
 
 func GetStorage(path string, args *GetStoragesArgs) (driver.Driver, error) {
 	storageDriver, _, err := op.GetStorageAndActualPath(path)
-	if err != nil && !args.NoLog {
+	if err != nil {
 		return nil, err
 	}
 	return storageDriver, nil

internal/fs/list.go

@@ -26,7 +26,9 @@ func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error)
 			ReqPath: path,
 		}, args.Refresh)
 		if err != nil {
-			log.Errorf("%+v", err)
+			if !args.NoLog {
+				log.Errorf("fs/list: %+v", err)
+			}
 			if len(virtualFiles) == 0 {
 				return nil, errors.WithMessage(err, "failed get objs")
 			}

pkg/task/task.go

@@ -3,6 +3,7 @@ package task
 
 import (
 	"context"
+	"runtime"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -63,11 +64,17 @@ func (t Task[K]) GetErrMsg() string {
 	return t.Error.Error()
 }
 
+func getCurrentGoroutineStack() string {
+	buf := make([]byte, 1<<16)
+	n := runtime.Stack(buf, false)
+	return string(buf[:n])
+}
+
 func (t *Task[K]) run() {
 	t.state = RUNNING
 	defer func() {
 		if err := recover(); err != nil {
-			log.Errorf("error [%+v] while run task [%s]", err, t.Name)
+			log.Errorf("error [%s] while run task [%s],stack trace:\n%s", err, t.Name, getCurrentGoroutineStack())
 			t.Error = errors.Errorf("panic: %+v", err)
 			t.state = ERRORED
 		}

server/handles/fsmanage.go

@@ -351,6 +351,105 @@ func FsRemove(c *gin.Context) {
 	common.SuccessResp(c)
 }
 
+type RemoveEmptyDirectoryReq struct {
+	SrcDir string `json:"src_dir"`
+}
+
+func FsRemoveEmptyDirectory(c *gin.Context) {
+	var req RemoveEmptyDirectoryReq
+	if err := c.ShouldBind(&req); err != nil {
+		common.ErrorResp(c, err, 400)
+		return
+	}
+
+	user := c.MustGet("user").(*model.User)
+	if !user.CanRemove() {
+		common.ErrorResp(c, errs.PermissionDenied, 403)
+		return
+	}
+	srcDir, err := user.JoinPath(req.SrcDir)
+	if err != nil {
+		common.ErrorResp(c, err, 403)
+		return
+	}
+
+	meta, err := op.GetNearestMeta(srcDir)
+	if err != nil {
+		if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
+			common.ErrorResp(c, err, 500, true)
+			return
+		}
+	}
+	c.Set("meta", meta)
+
+	rootFiles, err := fs.List(c, srcDir, &fs.ListArgs{})
+	if err != nil {
+		common.ErrorResp(c, err, 500)
+		return
+	}
+
+	// record the file path
+	filePathMap := make(map[model.Obj]string)
+	// record the parent file
+	fileParentMap := make(map[model.Obj]model.Obj)
+	// removing files
+	removingFiles := generic.NewQueue[model.Obj]()
+	// removed files
+	removedFiles := make(map[string]bool)
+	for _, file := range rootFiles {
+		if !file.IsDir() {
+			continue
+		}
+		removingFiles.Push(file)
+		filePathMap[file] = srcDir
+	}
+
+	for !removingFiles.IsEmpty() {
+
+		removingFile := removingFiles.Pop()
+		removingFilePath := fmt.Sprintf("%s/%s", filePathMap[removingFile], removingFile.GetName())
+
+		if removedFiles[removingFilePath] {
+			continue
+		}
+
+		subFiles, err := fs.List(c, removingFilePath, &fs.ListArgs{Refresh: true})
+		if err != nil {
+			common.ErrorResp(c, err, 500)
+			return
+		}
+
+		if len(subFiles) == 0 {
+			// remove empty directory
+			err = fs.Remove(c, removingFilePath)
+			removedFiles[removingFilePath] = true
+			if err != nil {
+				common.ErrorResp(c, err, 500)
+				return
+			}
+			// recheck parent folder
+			parentFile, exist := fileParentMap[removingFile]
+			if exist {
+				removingFiles.Push(parentFile)
+			}
+
+		} else {
+			// recursive remove
+			for _, subFile := range subFiles {
+				if !subFile.IsDir() {
+					continue
+				}
+				removingFiles.Push(subFile)
+				filePathMap[subFile] = removingFilePath
+				fileParentMap[subFile] = removingFile
+			}
+		}
+
+	}
+
+	common.SuccessResp(c)
+}
+
 // Link return real link, just for proxy program, it may contain cookie, so just allowed for admin
 func Link(c *gin.Context) {
 	var req MkdirOrLinkReq

server/router.go

@@ -131,6 +131,7 @@ func _fs(g *gin.RouterGroup) {
 	g.POST("/recursive_move", handles.FsRecursiveMove)
 	g.POST("/copy", handles.FsCopy)
 	g.POST("/remove", handles.FsRemove)
+	g.POST("/remove_empty_directory", handles.FsRemoveEmptyDirectory)
 	g.PUT("/put", middlewares.FsUp, handles.FsStream)
 	g.PUT("/form", middlewares.FsUp, handles.FsForm)
 	g.POST("/link", middlewares.AuthAdmin, handles.Link)