ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(alias): add DownloadConcurrency and DownloadPartSize option (#7829)

Browse Source 
* fix(net): goroutine logic bug (AlistGo/alist#7215)

* Fix goroutine logic bug

* Fix bug

---------

Co-authored-by: hpy hs <hshpy.pengyu@gmail.com>

* perf(net): sequential and dynamic concurrency

* fix(net): incorrect error return

* feat(alias):  add `DownloadConcurrency` and `DownloadPartSize` option

* feat(net): add `ConcurrencyLimit`

* pref(net): create `chunk` on demand

* refactor

* refactor

* fix(net): `r.Closers.Add` has no effect

* refactor

---------

Co-authored-by: hpy hs <hshpy.pengyu@gmail.com>
This commit is contained in:
j2rong4cn
2025-01-27 20:08:39 +08:00
committed by GitHub
parent bdcf450203
commit 2be0c3d1a0
24 changed files with 396 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
server/s3/backend.go
View File

@ -6,13 +6,14 @@ import (
"context"
"encoding/hex"
"fmt"
"github.com/pkg/errors"
"io"
"path"
"strings"
"sync"
"time"
"github.com/pkg/errors"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
@ -173,15 +174,27 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
if link.RangeReadCloser == nil && link.MFile == nil && len(link.URL) == 0 {
return nil, fmt.Errorf("the remote storage driver need to be enhanced to support s3")
}
remoteFileSize := file.GetSize()
remoteClosers := utils.EmptyClosers()
rangeReaderFunc := func(ctx context.Context, start, length int64) (io.ReadCloser, error) {
var rdr io.ReadCloser
length := int64(-1)
start := int64(0)
if rnge != nil {
start, length = rnge.Start, rnge.Length
}
// 参考 server/common/proxy.go
if link.MFile != nil {
_, err := link.MFile.Seek(start, io.SeekStart)
if err != nil {
return nil, err
}
rdr = link.MFile
} else {
remoteFileSize := file.GetSize()
if length >= 0 && start+length >= remoteFileSize {
length = -1
}
rrc := link.RangeReadCloser
if len(link.URL) > 0 {
rangedRemoteLink := &model.Link{
URL: link.URL,
Header: link.Header,
@ -194,35 +207,12 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
}
if rrc != nil {
remoteReader, err := rrc.RangeRead(ctx, http_range.Range{Start: start, Length: length})
remoteClosers.AddClosers(rrc.GetClosers())
if err != nil {
return nil, err
}
return remoteReader, nil
}
if link.MFile != nil {
_, err := link.MFile.Seek(start, io.SeekStart)
if err != nil {
return nil, err
}
//remoteClosers.Add(remoteLink.MFile)
//keep reuse same MFile and close at last.
remoteClosers.Add(link.MFile)
return io.NopCloser(link.MFile), nil
}
return nil, errs.NotSupport
}
var rdr io.ReadCloser
if rnge != nil {
rdr, err = rangeReaderFunc(ctx, rnge.Start, rnge.Length)
if err != nil {
return nil, err
}
} else {
rdr, err = rangeReaderFunc(ctx, 0, -1)
if err != nil {
return nil, err
rdr = utils.ReadCloser{Reader: remoteReader, Closer: rrc}
} else {
return nil, errs.NotSupport
}
}