ProjectHentai/alist
6
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(s3): server support generated url request (#6431)

Browse Source
This commit is contained in:
itsHenry
2024-05-09 14:28:59 +08:00
committed by GitHub
parent 2313213f59
commit 7e7b9b9b48
8 changed files with 27 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -3,11 +3,11 @@ module github.com/alist-org/alist/v3
go 1.21
require (
github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700
github.com/SheltonZhu/115driver v1.0.22
github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a
github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4
github.com/Xhofe/wopan-sdk-go v0.1.2
github.com/alist-org/gofakes3 v0.0.4
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
github.com/avast/retry-go v3.0.0+incompatible
github.com/aws/aws-sdk-go v1.50.24

4
go.sum
View File

@ -7,8 +7,6 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE=
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc=
github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 h1:r3fp2/Ro+0RtpjNY0/wsbN7vRmCW//dXTOZDQTct25Q=
github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY=
github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY=
github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
github.com/SheltonZhu/115driver v1.0.22 h1:Wp8pN7/gK3YwEO5P18ggbIOHM++lo9eP/pBhuvXfI6U=
@ -25,6 +23,8 @@ github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0E
github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ=
github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U=
github.com/alist-org/gofakes3 v0.0.4 h1:/ID4+1llsiB8EweLcC65rVmgBZKL95e3P7Wa+aJGUiE=
github.com/alist-org/gofakes3 v0.0.4/go.mod h1:bLPZXt45XYMgaoGGLe5t0d1p13oZTQTptTEDLrku070=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE=

44
server/s3/backend.go
View File

@ -12,7 +12,6 @@ import (
"sync"
"time"
"github.com/Mikubill/gofakes3"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
@ -20,12 +19,13 @@ import (
"github.com/alist-org/alist/v3/internal/stream"
"github.com/alist-org/alist/v3/pkg/http_range"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/alist-org/gofakes3"
"github.com/ncw/swift/v2"
)
var (
emptyPrefix = &gofakes3.Prefix{}
timeFormat = "Mon, 2 Jan 2006 15:04:05.999999999 GMT"
timeFormat = "Mon, 2 Jan 2006 15:04:05 GMT"
)
// s3Backend implements the gofacess3.Backend interface to make an S3
@ -42,13 +42,12 @@ func newBackend() gofakes3.Backend {
}
// ListBuckets always returns the default bucket.
func (b *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) {
func (b *s3Backend) ListBuckets(ctx context.Context) ([]gofakes3.BucketInfo, error) {
buckets, err := getAndParseBuckets()
if err != nil {
return nil, err
}
var response []gofakes3.BucketInfo
ctx := context.Background()
for _, b := range buckets {
node, _ := fs.Get(ctx, b.Path, &fs.GetArgs{})
response = append(response, gofakes3.BucketInfo{
@ -61,7 +60,7 @@ func (b *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) {
}
// ListBucket lists the objects in the given bucket.
func (b *s3Backend) ListBucket(bucketName string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) {
func (b *s3Backend) ListBucket(ctx context.Context, bucketName string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) {
bucket, err := getBucketByName(bucketName)
if err != nil {
return nil, err
@ -97,8 +96,7 @@ func (b *s3Backend) ListBucket(bucketName string, prefix *gofakes3.Prefix, page
// HeadObject returns the fileinfo for the given object name.
//
// Note that the metadata is not supported yet.
func (b *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object, error) {
ctx := context.Background()
func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName string) (*gofakes3.Object, error) {
bucket, err := getBucketByName(bucketName)
if err != nil {
return nil, err
@ -141,8 +139,7 @@ func (b *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object,
}
// GetObject fetchs the object from the filesystem.
func (b *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) {
ctx := context.Background()
func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) {
bucket, err := getBucketByName(bucketName)
if err != nil {
return nil, err
@ -251,18 +248,17 @@ func (b *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofak
}
// TouchObject creates or updates meta on specified object.
func (b *s3Backend) TouchObject(fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {
func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {
//TODO: implement
return result, gofakes3.ErrNotImplemented
}
// PutObject creates or overwrites the object with the given name.
func (b *s3Backend) PutObject(
bucketName, objectName string,
ctx context.Context, bucketName, objectName string,
meta map[string]string,
input io.Reader, size int64,
) (result gofakes3.PutObjectResult, err error) {
ctx := context.Background()
bucket, err := getBucketByName(bucketName)
if err != nil {
return result, err
@ -316,9 +312,9 @@ func (b *s3Backend) PutObject(
}
// DeleteMulti deletes multiple objects in a single request.
func (b *s3Backend) DeleteMulti(bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) {
func (b *s3Backend) DeleteMulti(ctx context.Context, bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) {
for _, object := range objects {
if err := b.deleteObject(bucketName, object); err != nil {
if err := b.deleteObject(ctx, bucketName, object); err != nil {
utils.Log.Errorf("serve s3", "delete object failed: %v", err)
result.Error = append(result.Error, gofakes3.ErrorResult{
Code: gofakes3.ErrInternal,
@ -336,13 +332,12 @@ func (b *s3Backend) DeleteMulti(bucketName string, objects ...string) (result go
}
// DeleteObject deletes the object with the given name.
func (b *s3Backend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) {
return result, b.deleteObject(bucketName, objectName)
func (b *s3Backend) DeleteObject(ctx context.Context, bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) {
return result, b.deleteObject(ctx, bucketName, objectName)
}
// deleteObject deletes the object from the filesystem.
func (b *s3Backend) deleteObject(bucketName, objectName string) error {
ctx := context.Background()
func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName string) error {
bucket, err := getBucketByName(bucketName)
if err != nil {
return err
@ -362,17 +357,17 @@ func (b *s3Backend) deleteObject(bucketName, objectName string) error {
}
// CreateBucket creates a new bucket.
func (b *s3Backend) CreateBucket(name string) error {
func (b *s3Backend) CreateBucket(ctx context.Context, name string) error {
return gofakes3.ErrNotImplemented
}
// DeleteBucket deletes the bucket with the given name.
func (b *s3Backend) DeleteBucket(name string) error {
func (b *s3Backend) DeleteBucket(ctx context.Context, name string) error {
return gofakes3.ErrNotImplemented
}
// BucketExists checks if the bucket exists.
func (b *s3Backend) BucketExists(name string) (exists bool, err error) {
func (b *s3Backend) BucketExists(ctx context.Context, name string) (exists bool, err error) {
buckets, err := getAndParseBuckets()
if err != nil {
return false, err
@ -386,13 +381,12 @@ func (b *s3Backend) BucketExists(name string) (exists bool, err error) {
}
// CopyObject copy specified object from srcKey to dstKey.
func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) {
func (b *s3Backend) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) {
if srcBucket == dstBucket && srcKey == dstKey {
//TODO: update meta
return result, nil
}
ctx := context.Background()
srcB, err := getBucketByName(srcBucket)
if err != nil {
return result, err
@ -403,7 +397,7 @@ func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta
fmeta, _ := op.GetNearestMeta(srcFp)
srcNode, err := fs.Get(context.WithValue(ctx, "meta", fmeta), srcFp, &fs.GetArgs{})
c, err := b.GetObject(srcBucket, srcKey, nil)
c, err := b.GetObject(ctx, srcBucket, srcKey, nil)
if err != nil {
return
}
@ -420,7 +414,7 @@ func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta
meta["mtime"] = swift.TimeToFloatString(srcNode.ModTime())
}
_, err = b.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size)
_, err = b.PutObject(ctx, dstBucket, dstKey, meta, c.Contents, c.Size)
if err != nil {
return
}

2
server/s3/list.go
View File

@ -6,7 +6,7 @@ import (
"path"
"strings"
"github.com/Mikubill/gofakes3"
"github.com/alist-org/gofakes3"
)
func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error {

2
server/s3/logger.go
View File

@ -5,8 +5,8 @@ package s3
import (
"fmt"
"github.com/Mikubill/gofakes3"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/alist-org/gofakes3"
)
// logger output formatted message

2
server/s3/pager.go
View File

@ -5,7 +5,7 @@ package s3
import (
"sort"
"github.com/Mikubill/gofakes3"
"github.com/alist-org/gofakes3"
)
// pager splits the object list into smulitply pages.

2
server/s3/server.go
View File

@ -7,7 +7,7 @@ import (
"math/rand"
"net/http"
"github.com/Mikubill/gofakes3"
"github.com/alist-org/gofakes3"
)
// Make a new S3 Server to serve the remote

2
server/s3/utils.go
View File

@ -7,13 +7,13 @@ import (
"encoding/json"
"strings"
"github.com/Mikubill/gofakes3"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/fs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/internal/setting"
"github.com/alist-org/gofakes3"
)
type Bucket struct {