Compare commits

...

26 Commits

Author SHA1 Message Date
2185839236 chore: safe base64 decode ipa name 2022-09-18 20:17:24 +08:00
24d58f278a fix: don't use cache if no objs 2022-09-18 18:38:47 +08:00
f80be96cf9 chore: replace sep _ with @ of ipa name 2022-09-18 16:53:39 +08:00
6c89c6c8ae fix: aria2 download magnet link (close #1665) 2022-09-18 16:07:32 +08:00
b74b55fa4a feat: support custom bundle-identifier by filename 2022-09-17 21:33:39 +08:00
09564102e7 fix(aliyundrive): rapid upload empty file (close #1699) 2022-09-17 19:39:19 +08:00
d436a6e676 fix: use base64 encode for ipa install 2022-09-17 17:06:08 +08:00
bec3a327a7 fix: hide objs if only virtual files 2022-09-17 15:31:30 +08:00
d329df70f3 fix: failed create record if use mysql (close #1690) 2022-09-16 22:21:43 +08:00
1af9f4061e fix(s3): remove folder recursively 2022-09-16 21:25:55 +08:00
0d012f85cb feat: Add thunderExpert priority video url switch 2022-09-15 22:50:27 +08:00
e3b213c398 feat: add ca-certificates for docker (fix: #1679) 2022-09-15 18:56:30 +08:00
d9f0603271 fix: copy folder between two storage (fix #1670) 2022-09-15 17:58:32 +08:00
86a625cb40 fix: set CHARSET to utf8mb4 if use mysql 2022-09-15 17:14:03 +08:00
f22232de5d chore: baidu_photo rename only duplicate folders 2022-09-15 09:25:20 +08:00
7ad3748a46 feat: update cache after remove instead of clear 2022-09-14 20:28:52 +08:00
66b2562d03 fix: allow force root while fetch dirs (close #1671) 2022-09-14 19:57:39 +08:00
b197322cd8 fix: type of file with name uppercase 2022-09-14 15:14:04 +08:00
9e5ef974a7 fix: send on closed channel 2022-09-14 15:13:02 +08:00
08a001fbd1 feat: add a start func for external calls (#1628) 2022-09-13 20:12:57 +08:00
54ae6dce0b fix(fs/get): rawURL if use proxy (close #1664) 2022-09-13 20:02:57 +08:00
a90ef201c7 fix(189pc,baidu_photo,thunder): single link limit multithreading 2022-09-13 18:44:07 +08:00
2de0da87fa fix: infinite loop if new multi-level folder (close #1661) 2022-09-13 18:34:04 +08:00
53e08e75fe fix(189pc,baidu_photo): source file not closed 2022-09-12 22:45:30 +08:00
6b5236f52e feat: add baidu_photo driver 2022-09-12 17:10:02 +08:00
78e34f0d9f fix: log error if err != nil (close #1651) 2022-09-12 17:01:06 +08:00
39 changed files with 1367 additions and 206 deletions

View File

@ -10,5 +10,6 @@ LABEL MAINTAINER="i@nn.ci"
VOLUME /opt/alist/data/
WORKDIR /opt/alist/
COPY --from=builder /app/bin/alist ./
RUN apk add ca-certificates
EXPOSE 5244
CMD [ "./alist", "server", "--no-prefix" ]

View File

@ -88,3 +88,12 @@ func init() {
// is called directly, e.g.:
// serverCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
// OutAlistInit 暴露用于外部启动server的函数
func OutAlistInit() {
var (
cmd *cobra.Command
args []string
)
serverCmd.Run(cmd, args)
}

View File

@ -3,8 +3,6 @@ package _189pc
import (
"context"
"net/http"
"regexp"
"strconv"
"strings"
"time"
@ -133,16 +131,17 @@ func (y *Yun189PC) Link(ctx context.Context, file model.Obj, args model.LinkArgs
"User-Agent": []string{base.UserAgent},
},
}
// 获取链接有效时常
strs := regexp.MustCompile(`(?i)expire[^=]*=([0-9]*)`).FindStringSubmatch(downloadUrl.URL)
if len(strs) == 2 {
timestamp, err := strconv.ParseInt(strs[1], 10, 64)
if err == nil {
expired := time.Duration(timestamp-time.Now().Unix()) * time.Second
like.Expiration = &expired
/*
// 获取链接有效时常
strs := regexp.MustCompile(`(?i)expire[^=]*=([0-9]*)`).FindStringSubmatch(downloadUrl.URL)
if len(strs) == 2 {
timestamp, err := strconv.ParseInt(strs[1], 10, 64)
if err == nil {
expired := time.Duration(timestamp-time.Now().Unix()) * time.Second
like.Expiration = &expired
}
}
}
*/
return like, nil
}

View File

@ -508,7 +508,6 @@ func (y *Yun189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.
_ = tempFile.Close()
_ = os.Remove(tempFile.Name())
}()
file.SetReadCloser(tempFile)
const DEFAULT int64 = 10485760
count := int(math.Ceil(float64(file.GetSize()) / float64(DEFAULT)))
@ -526,14 +525,16 @@ func (y *Yun189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.
}
silceMd5.Reset()
if _, err := io.CopyN(io.MultiWriter(fileMd5, silceMd5), file, DEFAULT); err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
if _, err := io.CopyN(io.MultiWriter(fileMd5, silceMd5), tempFile, DEFAULT); err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return err
}
md5Byte := silceMd5.Sum(nil)
silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Byte)))
silceMd5Base64s = append(silceMd5Base64s, fmt.Sprint(i, "-", base64.StdEncoding.EncodeToString(md5Byte)))
}
file.GetReadCloser().(*os.File).Seek(0, io.SeekStart)
if _, err = tempFile.Seek(0, io.SeekStart); err != nil {
return err
}
fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil)))
sliceMd5Hex := fileMd5Hex
@ -594,7 +595,7 @@ func (y *Yun189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.
SetContext(ctx).
SetQueryParams(clientSuffix()).
SetHeaders(ParseHttpHeader(uploadData.RequestHeader)).
SetBody(io.LimitReader(file, DEFAULT)).
SetBody(io.LimitReader(tempFile, DEFAULT)).
Put(uploadData.RequestURL)
if err != nil {
return err

View File

@ -234,7 +234,10 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileS
buf := make([]byte, 8)
r, _ := new(big.Int).SetString(utils.GetMD5Encode(d.AccessToken)[:16], 16)
i := new(big.Int).SetInt64(file.GetSize())
o := r.Mod(r, i)
o := new(big.Int).SetInt64(0)
if file.GetSize() > 0 {
o = r.Mod(r, i)
}
n, _ := io.NewSectionReader(tempFile, o.Int64(), 8).Read(buf[:8])
reqBody["proof_code"] = base64.StdEncoding.EncodeToString(buf[:n])

View File

@ -7,6 +7,7 @@ import (
_ "github.com/alist-org/alist/v3/drivers/189pc"
_ "github.com/alist-org/alist/v3/drivers/aliyundrive"
_ "github.com/alist-org/alist/v3/drivers/baidu_netdisk"
_ "github.com/alist-org/alist/v3/drivers/baidu_photo"
_ "github.com/alist-org/alist/v3/drivers/ftp"
_ "github.com/alist-org/alist/v3/drivers/google_drive"
_ "github.com/alist-org/alist/v3/drivers/local"

View File

@ -0,0 +1,282 @@
package baiduphoto
import (
"context"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"math"
"os"
"regexp"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/go-resty/resty/v2"
)
type BaiduPhoto struct {
model.Storage
Addition
AccessToken string
}
func (d *BaiduPhoto) Config() driver.Config {
return config
}
func (d *BaiduPhoto) GetAddition() driver.Additional {
return d.Addition
}
func (d *BaiduPhoto) Init(ctx context.Context, storage model.Storage) error {
d.Storage = storage
err := utils.Json.UnmarshalFromString(d.Storage.Addition, &d.Addition)
if err != nil {
return err
}
return d.refreshToken()
}
func (d *BaiduPhoto) Drop(ctx context.Context) error {
return nil
}
func (d *BaiduPhoto) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
var objs []model.Obj
var err error
if IsRoot(dir) {
var albums []Album
if d.ShowType != "root_only_file" {
albums, err = d.GetAllAlbum(ctx)
if err != nil {
return nil, err
}
}
var files []File
if d.ShowType != "root_only_album" {
files, err = d.GetAllFile(ctx)
if err != nil {
return nil, err
}
}
alubmName := make(map[string]int)
objs, _ = utils.SliceConvert(albums, func(album Album) (model.Obj, error) {
i := alubmName[album.GetName()]
if i != 0 {
alubmName[album.GetName()]++
album.Title = fmt.Sprintf("%s(%d)", album.Title, i)
}
alubmName[album.GetName()]++
return &album, nil
})
for i := 0; i < len(files); i++ {
objs = append(objs, &files[i])
}
} else if IsAlbum(dir) || IsAlbumRoot(dir) {
var files []AlbumFile
files, err = d.GetAllAlbumFile(ctx, splitID(dir.GetID())[0], "")
if err != nil {
return nil, err
}
objs = make([]model.Obj, 0, len(files))
for i := 0; i < len(files); i++ {
objs = append(objs, &files[i])
}
}
return objs, nil
}
func (d *BaiduPhoto) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
if IsAlbumFile(file) {
return d.linkAlbum(ctx, file, args)
} else if IsFile(file) {
return d.linkFile(ctx, file, args)
}
return nil, errs.NotFile
}
func (d *BaiduPhoto) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
if IsRoot(parentDir) {
code := regexp.MustCompile(`(?i)join:([\S]*)`).FindStringSubmatch(dirName)
if len(code) > 1 {
return d.JoinAlbum(ctx, code[1])
}
return d.CreateAlbum(ctx, dirName)
}
return errs.NotSupport
}
func (d *BaiduPhoto) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
if IsFile(srcObj) {
if IsAlbum(dstDir) {
//rootfile -> album
e := splitID(dstDir.GetID())
return d.AddAlbumFile(ctx, e[0], e[1], srcObj.GetID())
}
} else if IsAlbumFile(srcObj) {
if IsRoot(dstDir) {
//albumfile -> root
e := splitID(srcObj.GetID())
_, err := d.CopyAlbumFile(ctx, e[1], e[2], e[3], srcObj.GetID())
return err
} else if IsAlbum(dstDir) {
// albumfile -> root -> album
e := splitID(srcObj.GetID())
file, err := d.CopyAlbumFile(ctx, e[1], e[2], e[3], srcObj.GetID())
if err != nil {
return err
}
e = splitID(dstDir.GetID())
return d.AddAlbumFile(ctx, e[0], e[1], fmt.Sprint(file.Fsid))
}
}
return errs.NotSupport
}
func (d *BaiduPhoto) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
// 仅支持相册之间移动
if IsAlbumFile(srcObj) && IsAlbum(dstDir) {
err := d.Copy(ctx, srcObj, dstDir)
if err != nil {
return err
}
e := splitID(srcObj.GetID())
return d.DeleteAlbumFile(ctx, e[1], e[2], srcObj.GetID())
}
return errs.NotSupport
}
func (d *BaiduPhoto) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
// 仅支持相册改名
if IsAlbum(srcObj) {
e := splitID(srcObj.GetID())
return d.SetAlbumName(ctx, e[0], e[1], newName)
}
return errs.NotSupport
}
func (d *BaiduPhoto) Remove(ctx context.Context, obj model.Obj) error {
e := splitID(obj.GetID())
if IsFile(obj) {
return d.DeleteFile(ctx, e[0])
} else if IsAlbum(obj) {
return d.DeleteAlbum(ctx, e[0], e[1])
} else if IsAlbumFile(obj) {
return d.DeleteAlbumFile(ctx, e[1], e[2], obj.GetID())
}
return errs.NotSupport
}
func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
// 需要获取完整文件md5,必须支持 io.Seek
tempFile, err := utils.CreateTempFile(stream.GetReadCloser())
if err != nil {
return err
}
defer func() {
_ = tempFile.Close()
_ = os.Remove(tempFile.Name())
}()
// 计算需要的数据
const DEFAULT = 1 << 22
const SliceSize = 1 << 18
count := int(math.Ceil(float64(stream.GetSize()) / float64(DEFAULT)))
sliceMD5List := make([]string, 0, count)
fileMd5 := md5.New()
sliceMd5 := md5.New()
sliceMd52 := md5.New()
slicemd52Write := utils.LimitWriter(sliceMd52, SliceSize)
for i := 1; i <= count; i++ {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
_, err := io.CopyN(io.MultiWriter(fileMd5, sliceMd5, slicemd52Write), tempFile, DEFAULT)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return err
}
sliceMD5List = append(sliceMD5List, hex.EncodeToString(sliceMd5.Sum(nil)))
sliceMd5.Reset()
}
if _, err = tempFile.Seek(0, io.SeekStart); err != nil {
return err
}
content_md5 := hex.EncodeToString(fileMd5.Sum(nil))
slice_md5 := hex.EncodeToString(sliceMd52.Sum(nil))
// 开始执行上传
params := map[string]string{
"autoinit": "1",
"isdir": "0",
"rtype": "1",
"ctype": "11",
"path": stream.GetName(),
"size": fmt.Sprint(stream.GetSize()),
"slice-md5": slice_md5,
"content-md5": content_md5,
"block_list": MustString(utils.Json.MarshalToString(sliceMD5List)),
}
// 预上传
var precreateResp PrecreateResp
_, err = d.Post(FILE_API_URL_V1+"/precreate", func(r *resty.Request) {
r.SetContext(ctx)
r.SetFormData(params)
}, &precreateResp)
if err != nil {
return err
}
switch precreateResp.ReturnType {
case 1: // 上传文件
uploadParams := map[string]string{
"method": "upload",
"path": params["path"],
"uploadid": precreateResp.UploadID,
}
for i := 0; i < count; i++ {
uploadParams["partseq"] = fmt.Sprint(i)
_, err = d.Post("https://c3.pcs.baidu.com/rest/2.0/pcs/superfile2", func(r *resty.Request) {
r.SetContext(ctx)
r.SetQueryParams(uploadParams)
r.SetFileReader("file", stream.GetName(), io.LimitReader(tempFile, DEFAULT))
}, nil)
if err != nil {
return err
}
up(i * 100 / count)
}
fallthrough
case 2: // 创建文件
params["uploadid"] = precreateResp.UploadID
_, err = d.Post(FILE_API_URL_V1+"/create", func(r *resty.Request) {
r.SetContext(ctx)
r.SetFormData(params)
}, &precreateResp)
if err != nil {
return err
}
fallthrough
case 3: // 增加到相册
if IsAlbum(dstDir) || IsAlbumRoot(dstDir) {
e := splitID(dstDir.GetID())
err = d.AddAlbumFile(ctx, e[0], e[1], fmt.Sprint(precreateResp.Data.FsID))
if err != nil {
return err
}
}
}
return nil
}
var _ driver.Driver = (*BaiduPhoto)(nil)

107
drivers/baidu_photo/help.go Normal file
View File

@ -0,0 +1,107 @@
package baiduphoto
import (
"fmt"
"math"
"math/rand"
"regexp"
"strings"
"time"
"github.com/alist-org/alist/v3/internal/model"
)
//Tid生成
func getTid() string {
return fmt.Sprintf("3%d%.0f", time.Now().Unix(), math.Floor(9000000*rand.Float64()+1000000))
}
// 检查名称
func checkName(name string) bool {
return len(name) <= 20 && regexp.MustCompile("[\u4e00-\u9fa5A-Za-z0-9_-]").MatchString(name)
}
func toTime(t int64) *time.Time {
tm := time.Unix(t, 0)
return &tm
}
func fsidsFormat(ids ...string) string {
var buf []string
for _, id := range ids {
e := splitID(id)
buf = append(buf, fmt.Sprintf(`{"fsid":%s,"uk":%s}`, e[0], e[3]))
}
return fmt.Sprintf("[%s]", strings.Join(buf, ","))
}
func fsidsFormatNotUk(ids ...string) string {
var buf []string
for _, id := range ids {
buf = append(buf, fmt.Sprintf(`{"fsid":%s}`, splitID(id)[0]))
}
return fmt.Sprintf("[%s]", strings.Join(buf, ","))
}
/*
结构
{fsid} 文件
{album_id}|{tid} 相册
{fsid}|{album_id}|{tid}|{uk} 相册文件
*/
func splitID(id string) []string {
return strings.SplitN(id, "|", 4)[:4]
}
/*
结构
{fsid} 文件
{album_id}|{tid} 相册
{fsid}|{album_id}|{tid}|{uk} 相册文件
*/
func joinID(ids ...interface{}) string {
idsStr := make([]string, 0, len(ids))
for _, id := range ids {
idsStr = append(idsStr, fmt.Sprint(id))
}
return strings.Join(idsStr, "|")
}
func getFileName(path string) string {
return path[strings.LastIndex(path, "/")+1:]
}
// 相册
func IsAlbum(obj model.Obj) bool {
return obj.IsDir() && obj.GetPath() == "album"
}
// 根目录
func IsRoot(obj model.Obj) bool {
return obj.IsDir() && obj.GetPath() == "" && obj.GetID() == ""
}
// 以相册为根目录
func IsAlbumRoot(obj model.Obj) bool {
return obj.IsDir() && obj.GetPath() == "" && obj.GetID() != ""
}
// 根文件
func IsFile(obj model.Obj) bool {
return !obj.IsDir() && obj.GetPath() == "file"
}
// 相册文件
func IsAlbumFile(obj model.Obj) bool {
return !obj.IsDir() && obj.GetPath() == "albumfile"
}
func MustString(str string, err error) string {
return str
}

View File

@ -0,0 +1,30 @@
package baiduphoto
import (
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/op"
)
type Addition struct {
RefreshToken string `json:"refresh_token" required:"true"`
ShowType string `json:"show_type" type:"select" options:"root,root_only_album,root_only_file" default:"root"`
AlbumID string `json:"album_id"`
//AlbumPassword string `json:"album_password"`
ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"`
ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"`
}
func (a Addition) GetRootId() string {
return a.AlbumID
}
var config = driver.Config{
Name: "BaiduPhoto",
LocalSort: true,
}
func init() {
op.RegisterDriver(config, func() driver.Driver {
return &BaiduPhoto{}
})
}

View File

@ -0,0 +1,169 @@
package baiduphoto
import (
"fmt"
"time"
)
type TokenErrResp struct {
ErrorDescription string `json:"error_description"`
ErrorMsg string `json:"error"`
}
func (e *TokenErrResp) Error() string {
return fmt.Sprint(e.ErrorMsg, " : ", e.ErrorDescription)
}
type Erron struct {
Errno int `json:"errno"`
RequestID int `json:"request_id"`
}
type Page struct {
HasMore int `json:"has_more"`
Cursor string `json:"cursor"`
}
func (p Page) HasNextPage() bool {
return p.HasMore == 1
}
type (
FileListResp struct {
Page
List []File `json:"list"`
}
File struct {
Fsid int64 `json:"fsid"` // 文件ID
Path string `json:"path"` // 文件路径
Size int64 `json:"size"`
Ctime int64 `json:"ctime"` // 创建时间 s
Mtime int64 `json:"mtime"` // 修改时间 s
Thumburl []string `json:"thumburl"`
parseTime *time.Time
}
)
func (c *File) GetSize() int64 { return c.Size }
func (c *File) GetName() string { return getFileName(c.Path) }
func (c *File) ModTime() time.Time {
if c.parseTime == nil {
c.parseTime = toTime(c.Mtime)
}
return *c.parseTime
}
func (c *File) IsDir() bool { return false }
func (c *File) GetID() string { return joinID(c.Fsid) }
func (c *File) GetPath() string { return "file" }
func (c *File) Thumb() string {
if len(c.Thumburl) > 0 {
return c.Thumburl[0]
}
return ""
}
/*相册部分*/
type (
AlbumListResp struct {
Page
List []Album `json:"list"`
Reset int64 `json:"reset"`
TotalCount int64 `json:"total_count"`
}
Album struct {
AlbumID string `json:"album_id"`
Tid int64 `json:"tid"`
Title string `json:"title"`
JoinTime int64 `json:"join_time"`
CreateTime int64 `json:"create_time"`
Mtime int64 `json:"mtime"`
parseTime *time.Time
}
AlbumFileListResp struct {
Page
List []AlbumFile `json:"list"`
Reset int64 `json:"reset"`
TotalCount int64 `json:"total_count"`
}
AlbumFile struct {
File
AlbumID string `json:"album_id"`
Tid int64 `json:"tid"`
Uk int64 `json:"uk"`
}
)
func (a *Album) GetSize() int64 { return 0 }
func (a *Album) GetName() string { return fmt.Sprint(a.Title) }
func (a *Album) ModTime() time.Time {
if a.parseTime == nil {
a.parseTime = toTime(a.Mtime)
}
return *a.parseTime
}
func (a *Album) IsDir() bool { return true }
func (a *Album) GetID() string { return joinID(a.AlbumID, a.Tid) }
func (a *Album) GetPath() string { return "album" }
func (af *AlbumFile) GetID() string { return joinID(af.Fsid, af.AlbumID, af.Tid, af.Uk) }
func (c *AlbumFile) GetPath() string { return "albumfile" }
type (
CopyFileResp struct {
List []CopyFile `json:"list"`
}
CopyFile struct {
FromFsid int64 `json:"from_fsid"` // 源ID
Fsid int64 `json:"fsid"` // 目标ID
Path string `json:"path"`
ShootTime int `json:"shoot_time"`
}
)
/*上传部分*/
type (
UploadFile struct {
FsID int64 `json:"fs_id"`
Size int64 `json:"size"`
Md5 string `json:"md5"`
ServerFilename string `json:"server_filename"`
Path string `json:"path"`
Ctime int `json:"ctime"`
Mtime int `json:"mtime"`
Isdir int `json:"isdir"`
Category int `json:"category"`
ServerMd5 string `json:"server_md5"`
ShootTime int `json:"shoot_time"`
}
CreateFileResp struct {
Data UploadFile `json:"data"`
}
PrecreateResp struct {
ReturnType int `json:"return_type"` //存在返回2 不存在返回1 已经保存3
//存在返回
CreateFileResp
//不存在返回
Path string `json:"path"`
UploadID string `json:"uploadid"`
Blocklist []int64 `json:"block_list"`
}
)
type InviteResp struct {
Pdata struct {
// 邀请码
InviteCode string `json:"invite_code"`
// 有效时间
ExpireTime int `json:"expire_time"`
ShareID string `json:"share_id"`
} `json:"pdata"`
}

View File

@ -0,0 +1,376 @@
package baiduphoto
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/errs"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/go-resty/resty/v2"
)
const (
API_URL = "https://photo.baidu.com/youai"
ALBUM_API_URL = API_URL + "/album/v1"
FILE_API_URL_V1 = API_URL + "/file/v1"
FILE_API_URL_V2 = API_URL + "/file/v2"
)
var (
ErrNotSupportName = errors.New("only chinese and english, numbers and underscores are supported, and the length is no more than 20")
)
func (p *BaiduPhoto) Request(furl string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) {
req := base.RestyClient.R().
SetQueryParam("access_token", p.AccessToken)
if callback != nil {
callback(req)
}
if resp != nil {
req.SetResult(resp)
}
res, err := req.Execute(method, furl)
if err != nil {
return nil, err
}
erron := utils.Json.Get(res.Body(), "errno").ToInt()
switch erron {
case 0:
break
case 50805:
return nil, fmt.Errorf("you have joined album")
case 50820:
return nil, fmt.Errorf("no shared albums found")
case -6:
if err = p.refreshToken(); err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("errno: %d, refer to https://photo.baidu.com/union/doc", erron)
}
return res.Body(), nil
}
func (p *BaiduPhoto) refreshToken() error {
u := "https://openapi.baidu.com/oauth/2.0/token"
var resp base.TokenResp
var e TokenErrResp
_, err := base.RestyClient.R().SetResult(&resp).SetError(&e).SetQueryParams(map[string]string{
"grant_type": "refresh_token",
"refresh_token": p.RefreshToken,
"client_id": p.ClientID,
"client_secret": p.ClientSecret,
}).Get(u)
if err != nil {
return err
}
if e.ErrorMsg != "" {
return &e
}
if resp.RefreshToken == "" {
return errs.EmptyToken
}
p.AccessToken, p.RefreshToken = resp.AccessToken, resp.RefreshToken
op.MustSaveDriverStorage(p)
return nil
}
func (p *BaiduPhoto) Get(furl string, callback base.ReqCallback, resp interface{}) ([]byte, error) {
return p.Request(furl, http.MethodGet, callback, resp)
}
func (p *BaiduPhoto) Post(furl string, callback base.ReqCallback, resp interface{}) ([]byte, error) {
return p.Request(furl, http.MethodPost, callback, resp)
}
// 获取所有文件
func (p *BaiduPhoto) GetAllFile(ctx context.Context) (files []File, err error) {
var cursor string
for {
var resp FileListResp
_, err = p.Get(FILE_API_URL_V1+"/list", func(r *resty.Request) {
r.SetContext(ctx)
r.SetQueryParams(map[string]string{
"need_thumbnail": "1",
"need_filter_hidden": "0",
"cursor": cursor,
})
}, &resp)
if err != nil {
return
}
files = append(files, resp.List...)
if !resp.HasNextPage() {
return
}
cursor = resp.Cursor
}
}
// 删除根文件
func (p *BaiduPhoto) DeleteFile(ctx context.Context, fileIDs ...string) error {
_, err := p.Get(FILE_API_URL_V1+"/delete", func(req *resty.Request) {
req.SetContext(ctx)
req.SetQueryParams(map[string]string{
"fsid_list": fmt.Sprintf("[%s]", strings.Join(fileIDs, ",")),
})
}, nil)
return err
}
// 获取所有相册
func (p *BaiduPhoto) GetAllAlbum(ctx context.Context) (albums []Album, err error) {
var cursor string
for {
var resp AlbumListResp
_, err = p.Get(ALBUM_API_URL+"/list", func(r *resty.Request) {
r.SetContext(ctx)
r.SetQueryParams(map[string]string{
"need_amount": "1",
"limit": "100",
"cursor": cursor,
})
}, &resp)
if err != nil {
return
}
if albums == nil {
albums = make([]Album, 0, resp.TotalCount)
}
cursor = resp.Cursor
albums = append(albums, resp.List...)
if !resp.HasNextPage() {
return
}
}
}
// 获取相册中所有文件
func (p *BaiduPhoto) GetAllAlbumFile(ctx context.Context, albumID, passwd string) (files []AlbumFile, err error) {
var cursor string
for {
var resp AlbumFileListResp
_, err = p.Get(ALBUM_API_URL+"/listfile", func(r *resty.Request) {
r.SetContext(ctx)
r.SetQueryParams(map[string]string{
"album_id": albumID,
"need_amount": "1",
"limit": "1000",
"passwd": passwd,
"cursor": cursor,
})
}, &resp)
if err != nil {
return
}
if files == nil {
files = make([]AlbumFile, 0, resp.TotalCount)
}
cursor = resp.Cursor
files = append(files, resp.List...)
if !resp.HasNextPage() {
return
}
}
}
// 创建相册
func (p *BaiduPhoto) CreateAlbum(ctx context.Context, name string) error {
if !checkName(name) {
return ErrNotSupportName
}
_, err := p.Post(ALBUM_API_URL+"/create", func(r *resty.Request) {
r.SetContext(ctx)
r.SetQueryParams(map[string]string{
"title": name,
"tid": getTid(),
"source": "0",
})
}, nil)
return err
}
// 相册改名
func (p *BaiduPhoto) SetAlbumName(ctx context.Context, albumID, tID, name string) error {
if !checkName(name) {
return ErrNotSupportName
}
_, err := p.Post(ALBUM_API_URL+"/settitle", func(r *resty.Request) {
r.SetContext(ctx)
r.SetFormData(map[string]string{
"title": name,
"album_id": albumID,
"tid": tID,
})
}, nil)
return err
}
// 删除相册
func (p *BaiduPhoto) DeleteAlbum(ctx context.Context, albumID, tID string) error {
_, err := p.Post(ALBUM_API_URL+"/delete", func(r *resty.Request) {
r.SetContext(ctx)
r.SetFormData(map[string]string{
"album_id": albumID,
"tid": tID,
"delete_origin_image": "0", // 是否删除原图 0 不删除 1 删除
})
}, nil)
return err
}
// 删除相册文件
func (p *BaiduPhoto) DeleteAlbumFile(ctx context.Context, albumID, tID string, fileIDs ...string) error {
_, err := p.Post(ALBUM_API_URL+"/delfile", func(r *resty.Request) {
r.SetContext(ctx)
r.SetFormData(map[string]string{
"album_id": albumID,
"tid": tID,
"list": fsidsFormat(fileIDs...),
"del_origin": "0", // 是否删除原图 0 不删除 1 删除
})
}, nil)
return err
}
// 增加相册文件
func (p *BaiduPhoto) AddAlbumFile(ctx context.Context, albumID, tID string, fileIDs ...string) error {
_, err := p.Get(ALBUM_API_URL+"/addfile", func(r *resty.Request) {
r.SetContext(ctx)
r.SetQueryParams(map[string]string{
"album_id": albumID,
"tid": tID,
"list": fsidsFormatNotUk(fileIDs...),
})
}, nil)
return err
}
// 保存相册文件为根文件
func (p *BaiduPhoto) CopyAlbumFile(ctx context.Context, albumID, tID, uk string, fileID ...string) (*CopyFile, error) {
var resp CopyFileResp
_, err := p.Post(ALBUM_API_URL+"/copyfile", func(r *resty.Request) {
r.SetContext(ctx)
r.SetFormData(map[string]string{
"album_id": albumID,
"tid": tID,
"uk": uk,
"list": fsidsFormatNotUk(fileID...),
})
r.SetResult(&resp)
}, nil)
if err != nil {
return nil, err
}
return &resp.List[0], nil
}
// 加入相册
func (p *BaiduPhoto) JoinAlbum(ctx context.Context, code string) error {
var resp InviteResp
_, err := p.Get(ALBUM_API_URL+"/querypcode", func(req *resty.Request) {
req.SetContext(ctx)
req.SetQueryParams(map[string]string{
"pcode": code,
"web": "1",
})
}, &resp)
if err != nil {
return err
}
_, err = p.Get(ALBUM_API_URL+"/join", func(req *resty.Request) {
req.SetContext(ctx)
req.SetQueryParams(map[string]string{
"invite_code": resp.Pdata.InviteCode,
})
}, nil)
return err
}
func (d *BaiduPhoto) linkAlbum(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
headers := map[string]string{
"User-Agent": base.UserAgent,
}
if args.Header.Get("User-Agent") != "" {
headers["User-Agent"] = args.Header.Get("User-Agent")
}
if !utils.IsLocalIPAddr(args.IP) {
headers["X-Forwarded-For"] = args.IP
}
e := splitID(file.GetID())
res, err := base.NoRedirectClient.R().
SetContext(ctx).
SetHeaders(headers).
SetQueryParams(map[string]string{
"access_token": d.AccessToken,
"fsid": e[0],
"album_id": e[1],
"tid": e[2],
"uk": e[3],
}).
Head(ALBUM_API_URL + "/download")
if err != nil {
return nil, err
}
//exp := 8 * time.Hour
link := &model.Link{
URL: res.Header().Get("location"),
Header: http.Header{
"User-Agent": []string{headers["User-Agent"]},
},
//Expiration: &exp,
}
return link, nil
}
func (d *BaiduPhoto) linkFile(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
headers := map[string]string{
"User-Agent": base.UserAgent,
}
if args.Header.Get("User-Agent") != "" {
headers["User-Agent"] = args.Header.Get("User-Agent")
}
if !utils.IsLocalIPAddr(args.IP) {
headers["X-Forwarded-For"] = args.IP
}
var downloadUrl struct {
Dlink string `json:"dlink"`
}
_, err := d.Get(FILE_API_URL_V2+"/download", func(r *resty.Request) {
r.SetContext(ctx)
r.SetHeaders(headers)
r.SetQueryParams(map[string]string{
"fsid": splitID(file.GetID())[0],
})
}, &downloadUrl)
if err != nil {
return nil, err
}
//exp := 8 * time.Hour
link := &model.Link{
URL: downloadUrl.Dlink,
Header: http.Header{
"User-Agent": []string{headers["User-Agent"]},
},
//Expiration: &exp,
}
return link, nil
}

View File

@ -175,9 +175,9 @@ func (d *Local) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
dstPath := filepath.Join(dstDir.GetPath(), srcObj.GetName())
var err error
if srcObj.IsDir() {
err = copyDir(srcPath, dstPath)
err = utils.CopyDir(srcPath, dstPath)
} else {
err = copyFile(srcPath, dstPath)
err = utils.CopyFile(srcPath, dstPath)
}
if err != nil {
return err

View File

@ -1,67 +1 @@
package local
import (
"fmt"
"io"
"io/ioutil"
"os"
"path"
)
// copyFile File copies a single file from src to dst
func copyFile(src, dst string) error {
var err error
var srcfd *os.File
var dstfd *os.File
var srcinfo os.FileInfo
if srcfd, err = os.Open(src); err != nil {
return err
}
defer srcfd.Close()
if dstfd, err = os.Create(dst); err != nil {
return err
}
defer dstfd.Close()
if _, err = io.Copy(dstfd, srcfd); err != nil {
return err
}
if srcinfo, err = os.Stat(src); err != nil {
return err
}
return os.Chmod(dst, srcinfo.Mode())
}
// copyDir Dir copies a whole directory recursively
func copyDir(src string, dst string) error {
var err error
var fds []os.FileInfo
var srcinfo os.FileInfo
if srcinfo, err = os.Stat(src); err != nil {
return err
}
if err = os.MkdirAll(dst, srcinfo.Mode()); err != nil {
return err
}
if fds, err = ioutil.ReadDir(src); err != nil {
return err
}
for _, fd := range fds {
srcfp := path.Join(src, fd.Name())
dstfp := path.Join(dst, fd.Name())
if fd.IsDir() {
if err = copyDir(srcfp, dstfp); err != nil {
fmt.Println(err)
}
} else {
if err = copyFile(srcfp, dstfp); err != nil {
fmt.Println(err)
}
}
}
return nil
}

View File

@ -8,6 +8,7 @@ import (
"io"
"net/http"
"os"
"time"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/driver"
@ -90,6 +91,9 @@ func (d *Quark) MakeDir(ctx context.Context, parentDir model.Obj, dirName string
_, err := d.request("/file", http.MethodPost, func(req *resty.Request) {
req.SetBody(data)
}, nil)
if err == nil {
time.Sleep(time.Second)
}
return err
}

View File

@ -130,13 +130,10 @@ func (d *S3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
}
func (d *S3) Remove(ctx context.Context, obj model.Obj) error {
key := getKey(obj.GetPath(), obj.IsDir())
input := &s3.DeleteObjectInput{
Bucket: &d.Bucket,
Key: &key,
if obj.IsDir() {
return d.removeDir(ctx, obj.GetPath())
}
_, err := d.client.DeleteObject(input)
return err
return d.removeFile(obj.GetPath())
}
func (d *S3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {

View File

@ -52,12 +52,11 @@ func getKey(path string, dir bool) string {
return path
}
var defaultPlaceholderName = ".placeholder"
// var defaultPlaceholderName = ".placeholder"
func getPlaceholderName(placeholder string) string {
if placeholder == "" {
return defaultPlaceholderName
}
//if placeholder == "" {
// return defaultPlaceholderName
//}
return placeholder
}
@ -205,3 +204,33 @@ func (d *S3) copyDir(ctx context.Context, src string, dst string) error {
}
return nil
}
func (d *S3) removeDir(ctx context.Context, src string) error {
objs, err := op.List(ctx, d, src, model.ListArgs{})
if err != nil {
return err
}
for _, obj := range objs {
cSrc := path.Join(src, obj.GetName())
if obj.IsDir() {
err = d.removeDir(ctx, cSrc)
} else {
err = d.removeFile(cSrc)
}
if err != nil {
return err
}
}
_ = d.removeFile(path.Join(src, getPlaceholderName(d.Placeholder)))
return nil
}
func (d *S3) removeFile(src string) error {
key := getKey(src, true)
input := &s3.DeleteObjectInput{
Bucket: &d.Bucket,
Key: &key,
}
_, err := d.client.DeleteObject(input)
return err
}

View File

@ -4,10 +4,7 @@ import (
"context"
"fmt"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/driver"
@ -149,6 +146,7 @@ func (x *ThunderExpert) Init(ctx context.Context, storage model.Storage) (err er
PackageName: x.PackageName,
UserAgent: x.UserAgent,
DownloadUserAgent: x.DownloadUserAgent,
UseVideoUrl: x.UseVideoUrl,
},
}
@ -212,6 +210,7 @@ func (x *ThunderExpert) Init(ctx context.Context, storage model.Storage) (err er
}
x.XunLeiCommon.UserAgent = x.UserAgent
x.XunLeiCommon.DownloadUserAgent = x.DownloadUserAgent
x.XunLeiCommon.UseVideoUrl = x.UseVideoUrl
}
return nil
}
@ -255,14 +254,25 @@ func (xc *XunLeiCommon) Link(ctx context.Context, file model.Obj, args model.Lin
},
}
strs := regexp.MustCompile(`e=([0-9]*)`).FindStringSubmatch(lFile.WebContentLink)
if len(strs) == 2 {
timestamp, err := strconv.ParseInt(strs[1], 10, 64)
if err == nil {
expired := time.Duration(timestamp-time.Now().Unix()) * time.Second
link.Expiration = &expired
if xc.UseVideoUrl {
for _, media := range lFile.Medias {
if media.Link.URL != "" {
link.URL = media.Link.URL
break
}
}
}
/*
strs := regexp.MustCompile(`e=([0-9]*)`).FindStringSubmatch(lFile.WebContentLink)
if len(strs) == 2 {
timestamp, err := strconv.ParseInt(strs[1], 10, 64)
if err == nil {
expired := time.Duration(timestamp-time.Now().Unix()) * time.Second
link.Expiration = &expired
}
}
*/
return link, nil
}

View File

@ -41,6 +41,9 @@ type ExpertAddition struct {
//不影响登录,影响下载速度
UserAgent string `json:"user_agent" required:"true" default:"ANDROID-com.xunlei.downloadprovider/7.51.0.8196 netWorkType/4G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/220200 Oauth2Client/0.9 (Linux 4_14_186-perf-gdcf98eab238b) (JAVA 0)"`
DownloadUserAgent string `json:"download_user_agent" required:"true" default:"Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)"`
//优先使用视频链接代替下载链接
UseVideoUrl bool `json:"use_video_url"`
}
// 登录特征,用于判断是否重新登录

View File

@ -77,6 +77,13 @@ type FileList struct {
VersionOutdated bool `json:"version_outdated"`
}
type Link struct {
URL string `json:"url"`
Token string `json:"token"`
Expire time.Time `json:"expire"`
Type string `json:"type"`
}
type Files struct {
Kind string `json:"kind"`
ID string `json:"id"`
@ -95,26 +102,26 @@ type Files struct {
ThumbnailLink string `json:"thumbnail_link"`
//Md5Checksum string `json:"md5_checksum"`
//Hash string `json:"hash"`
//Links struct{} `json:"links"`
Phase string `json:"phase"`
Links map[string]Link `json:"links"`
Phase string `json:"phase"`
Audit struct {
Status string `json:"status"`
Message string `json:"message"`
Title string `json:"title"`
} `json:"audit"`
/* Medias []struct {
Category string `json:"category"`
IconLink string `json:"icon_link"`
IsDefault bool `json:"is_default"`
IsOrigin bool `json:"is_origin"`
IsVisible bool `json:"is_visible"`
//Link interface{} `json:"link"`
MediaID string `json:"media_id"`
MediaName string `json:"media_name"`
NeedMoreQuota bool `json:"need_more_quota"`
Priority int `json:"priority"`
RedirectLink string `json:"redirect_link"`
ResolutionName string `json:"resolution_name"`
Medias []struct {
Category string `json:"category"`
IconLink string `json:"icon_link"`
IsDefault bool `json:"is_default"`
IsOrigin bool `json:"is_origin"`
IsVisible bool `json:"is_visible"`
Link Link `json:"link"`
MediaID string `json:"media_id"`
MediaName string `json:"media_name"`
NeedMoreQuota bool `json:"need_more_quota"`
Priority int `json:"priority"`
RedirectLink string `json:"redirect_link"`
ResolutionName string `json:"resolution_name"`
Video struct {
AudioCodec string `json:"audio_codec"`
BitRate int `json:"bit_rate"`
@ -126,7 +133,7 @@ type Files struct {
Width int `json:"width"`
} `json:"video"`
VipTypes []string `json:"vip_types"`
} `json:"medias"` */
} `json:"medias"`
Trashed bool `json:"trashed"`
DeleteTime string `json:"delete_time"`
OriginalURL string `json:"original_url"`

View File

@ -52,6 +52,7 @@ type Common struct {
PackageName string
UserAgent string
DownloadUserAgent string
UseVideoUrl bool
}
func (c *Common) SetCaptchaToken(captchaToken string) {

View File

@ -76,10 +76,15 @@ func (m *Monitor) Update() (bool, error) {
}
m.retried = 0
if len(info.FollowedBy) != 0 {
log.Debugf("followen by: %+v", info.FollowedBy)
gid := info.FollowedBy[0]
notify.Signals.Delete(m.tsk.ID)
oldId := m.tsk.ID
m.tsk.ID = gid
DownTaskManager.RawTasks().Delete(oldId)
DownTaskManager.RawTasks().Store(m.tsk.ID, m.tsk)
notify.Signals.Store(gid, m.c)
return false, nil
}
// update download status
total, err := strconv.ParseUint(info.TotalLength, 10, 64)
@ -120,6 +125,7 @@ func (m *Monitor) Complete() error {
}
// get files
files, err := client.GetFiles(m.tsk.ID)
log.Debugf("files len: %d", len(files))
if err != nil {
return errors.Wrapf(err, "failed to get files of %s", m.tsk.ID)
}
@ -134,7 +140,8 @@ func (m *Monitor) Complete() error {
log.Errorf("failed to remove aria2 temp dir: %+v", err.Error())
}
}()
for _, file := range files {
for i, _ := range files {
file := files[i]
TransferTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{
Name: fmt.Sprintf("transfer %s to [%s](%s)", file.Path, storage.GetStorage().MountPath, dstDirActualPath),
Func: func(tsk *task.Task[uint64]) error {

View File

@ -8,6 +8,8 @@ import (
func InitAria2() {
go func() {
_, err := aria2.InitClient(2)
utils.Log.Errorf("failed to init aria2 client: %+v", err)
if err != nil {
utils.Log.Errorf("failed to init aria2 client: %+v", err)
}
}()
}

View File

@ -3,15 +3,21 @@ package db
import (
"log"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/model"
"gorm.io/gorm"
)
var db gorm.DB
var db *gorm.DB
func Init(d *gorm.DB) {
db = *d
err := db.AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
db = d
var err error
if conf.Conf.Database.Type == "mysql" {
err = db.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
} else {
err = db.AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem))
}
if err != nil {
log.Fatalf("failed migrate database: %s", err.Error())
}

View File

@ -12,6 +12,7 @@ import (
"github.com/alist-org/alist/v3/pkg/task"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
var CopyTaskManager = task.NewTaskManager(3, func(tid *uint64) {
@ -60,7 +61,7 @@ func copyBetween2Storages(t *task.Task[uint64], srcStorage, dstStorage driver.Dr
return nil
}
srcObjPath := stdpath.Join(srcObjPath, obj.GetName())
dstObjPath := stdpath.Join(dstDirPath, obj.GetName())
dstObjPath := stdpath.Join(dstDirPath, srcObj.GetName())
CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{
Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().MountPath, srcObjPath, dstStorage.GetStorage().MountPath, dstObjPath),
Func: func(t *task.Task[uint64]) error {
@ -72,7 +73,9 @@ func copyBetween2Storages(t *task.Task[uint64], srcStorage, dstStorage driver.Dr
CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{
Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().MountPath, srcObjPath, dstStorage.GetStorage().MountPath, dstDirPath),
Func: func(t *task.Task[uint64]) error {
return copyFileBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstDirPath)
err := copyFileBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstDirPath)
log.Debugf("copy file between storages: %+v", err)
return err
},
}))
}

View File

@ -16,37 +16,43 @@ import (
func list(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error) {
meta := ctx.Value("meta").(*model.Meta)
user := ctx.Value("user").(*model.User)
var objs []model.Obj
storage, actualPath, err := op.GetStorageAndActualPath(path)
virtualFiles := op.GetStorageVirtualFilesByPath(path)
if err != nil {
if len(virtualFiles) != 0 {
return virtualFiles, nil
if len(virtualFiles) == 0 {
return nil, errors.WithMessage(err, "failed get storage")
}
return nil, errors.WithMessage(err, "failed get storage")
}
objs, err := op.List(ctx, storage, actualPath, model.ListArgs{
ReqPath: path,
}, refresh...)
if err != nil {
log.Errorf("%+v", err)
if len(virtualFiles) != 0 {
return virtualFiles, nil
} else {
objs, err = op.List(ctx, storage, actualPath, model.ListArgs{
ReqPath: path,
}, refresh...)
if err != nil {
log.Errorf("%+v", err)
if len(virtualFiles) == 0 {
return nil, errors.WithMessage(err, "failed get objs")
}
}
return nil, errors.WithMessage(err, "failed get objs")
}
for _, storageFile := range virtualFiles {
if !containsByName(objs, storageFile) {
objs = append(objs, storageFile)
if objs == nil {
objs = virtualFiles
} else {
for _, storageFile := range virtualFiles {
if !containsByName(objs, storageFile) {
objs = append(objs, storageFile)
}
}
}
if whetherHide(user, meta, path) {
objs = hide(objs, meta)
}
// sort objs
if storage.Config().LocalSort {
model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
if storage != nil {
if storage.Config().LocalSort {
model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)
}
model.ExtractFolder(objs, storage.GetStorage().ExtractFolder)
}
model.ExtractFolder(objs, storage.GetStorage().ExtractFolder)
return objs, nil
}

View File

@ -1,6 +1,7 @@
package fs
import (
"fmt"
"io"
"mime"
"net/http"
@ -8,8 +9,11 @@ import (
stdpath "path"
"strings"
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/google/uuid"
"github.com/pkg/errors"
)
@ -38,7 +42,13 @@ func getFileStreamFromLink(file model.Obj, link *model.Link) (model.FileStreamer
if link.Data != nil {
rc = link.Data
} else if link.FilePath != nil {
f, err := os.Open(*link.FilePath)
// copy a new temp, because will be deleted after upload
newFilePath := stdpath.Join(conf.Conf.TempDir, fmt.Sprintf("%s-%s", uuid.NewString(), file.GetName()))
err := utils.CopyFile(*link.FilePath, newFilePath)
if err != nil {
return nil, err
}
f, err := os.Open(newFilePath)
if err != nil {
return nil, errors.Wrapf(err, "failed to open file %s", *link.FilePath)
}

View File

@ -27,6 +27,10 @@ func ClearCache(storage driver.Driver, path string) {
listCache.Del(key)
}
func Key(storage driver.Driver, path string) string {
return stdpath.Join(storage.GetStorage().MountPath, utils.StandardizePath(path))
}
// List files in storage, not contains virtual file
func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
@ -46,9 +50,9 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
objs, err := storage.List(ctx, dir, args)
return objs, errors.WithStack(err)
}
key := stdpath.Join(storage.GetStorage().MountPath, path)
key := Key(storage, path)
if len(refresh) == 0 || !refresh[0] {
if files, ok := listCache.Get(key); ok {
if files, ok := listCache.Get(key); ok && len(files) > 0 {
return files, nil
}
}
@ -181,6 +185,7 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
return errors.Errorf("storage not init: %s", storage.GetStorage().Status)
}
path = utils.StandardizePath(path)
// check if dir exists
f, err := Get(ctx, storage, path)
if err != nil {
@ -195,7 +200,11 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error {
if err != nil {
return errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath)
}
return errors.WithStack(storage.MakeDir(ctx, parentDir, dirName))
err = storage.MakeDir(ctx, parentDir, dirName)
if err == nil {
ClearCache(storage, parentPath)
}
return errors.WithStack(err)
} else {
return errors.WithMessage(err, "failed to check if dir exists")
}
@ -261,7 +270,28 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error {
}
return errors.WithMessage(err, "failed to get object")
}
return errors.WithStack(storage.Remove(ctx, obj))
err = storage.Remove(ctx, obj)
if err == nil {
key := Key(storage, stdpath.Dir(path))
if objs, ok := listCache.Get(key); ok {
j := -1
for i, m := range objs {
if m.GetName() == obj.GetName() {
j = i
break
}
}
if j >= 0 && j < len(objs) {
objs = append(objs[:j], objs[j+1:]...)
listCache.Set(key, objs)
} else {
log.Debugf("not found obj")
}
} else {
log.Debugf("not found parent cache")
}
}
return errors.WithStack(err)
}
func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error {
@ -308,12 +338,10 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod
}
err = storage.Put(ctx, parentDir, file, up)
log.Debugf("put file [%s] done", file.GetName())
if err == nil {
// set as complete
up(100)
// clear cache
key := stdpath.Join(storage.GetStorage().MountPath, dstDirPath)
listCache.Del(key)
}
//if err == nil {
// //clear cache
// key := stdpath.Join(storage.GetStorage().MountPath, dstDirPath)
// listCache.Del(key)
//}
return errors.WithStack(err)
}

View File

@ -94,19 +94,14 @@ func EnableStorage(ctx context.Context, id uint) error {
if !storage.Disabled {
return errors.Errorf("this storage have enabled")
}
err = LoadStorage(ctx, *storage)
if err != nil {
return errors.WithMessage(err, "failed load storage")
}
// re-get storage from db, because it maybe hava updated
storage, err = db.GetStorageById(id)
if err != nil {
return errors.WithMessage(err, "failed re-get storage again")
}
storage.Disabled = false
err = db.UpdateStorage(storage)
if err != nil {
return errors.WithMessage(err, "failed update storage in db, but have load in memory. you can try restart")
return errors.WithMessage(err, "failed update storage in db")
}
err = LoadStorage(ctx, *storage)
if err != nil {
return errors.WithMessage(err, "failed load storage")
}
return nil
}
@ -128,12 +123,12 @@ func DisableStorage(ctx context.Context, id uint) error {
return errors.Wrapf(err, "failed drop storage")
}
// delete the storage in the memory
storagesMap.Delete(storage.MountPath)
storage.Disabled = true
err = db.UpdateStorage(storage)
if err != nil {
return errors.WithMessage(err, "failed update storage in db, but have drop in memory. you can try restart")
return errors.WithMessage(err, "failed update storage in db")
}
storagesMap.Delete(storage.MountPath)
return nil
}

View File

@ -30,6 +30,10 @@ func (c *Cron) Do(f func()) {
}
func (c *Cron) Stop() {
c.ch <- struct{}{}
close(c.ch)
select {
case _, _ = <-c.ch:
default:
c.ch <- struct{}{}
close(c.ch)
}
}

View File

@ -10,6 +10,7 @@ func TestCron(t *testing.T) {
c.Do(func() {
t.Logf("cron log")
})
time.Sleep(time.Second * 5)
time.Sleep(time.Second * 3)
c.Stop()
c.Stop()
}

View File

@ -122,6 +122,10 @@ func (tm *Manager[K]) ClearDone() {
tm.RemoveByStates(SUCCEEDED, CANCELED, ERRORED)
}
func (tm *Manager[K]) RawTasks() *generic_sync.MapOf[K, *Task[K]] {
return &tm.tasks
}
func NewTaskManager[K comparable](maxWorker int, updateID ...func(*K)) *Manager[K] {
tm := &Manager[K]{
tasks: generic_sync.MapOf[K, *Task[K]]{},

View File

@ -82,6 +82,7 @@ func (t *Task[K]) run() {
t.state = ERRORED
} else {
t.state = SUCCEEDED
t.SetProgress(100)
if t.callback != nil {
t.callback(t)
}

View File

@ -1,14 +1,76 @@
package utils
import (
"fmt"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"github.com/alist-org/alist/v3/internal/conf"
log "github.com/sirupsen/logrus"
)
// CopyFile File copies a single file from src to dst
func CopyFile(src, dst string) error {
var err error
var srcfd *os.File
var dstfd *os.File
var srcinfo os.FileInfo
if srcfd, err = os.Open(src); err != nil {
return err
}
defer srcfd.Close()
if dstfd, err = CreateNestedFile(dst); err != nil {
return err
}
defer dstfd.Close()
if _, err = io.Copy(dstfd, srcfd); err != nil {
return err
}
if srcinfo, err = os.Stat(src); err != nil {
return err
}
return os.Chmod(dst, srcinfo.Mode())
}
// CopyDir Dir copies a whole directory recursively
func CopyDir(src string, dst string) error {
var err error
var fds []os.FileInfo
var srcinfo os.FileInfo
if srcinfo, err = os.Stat(src); err != nil {
return err
}
if err = os.MkdirAll(dst, srcinfo.Mode()); err != nil {
return err
}
if fds, err = ioutil.ReadDir(src); err != nil {
return err
}
for _, fd := range fds {
srcfp := path.Join(src, fd.Name())
dstfp := path.Join(dst, fd.Name())
if fd.IsDir() {
if err = CopyDir(srcfp, dstfp); err != nil {
fmt.Println(err)
}
} else {
if err = CopyFile(srcfp, dstfp); err != nil {
fmt.Println(err)
}
}
}
return nil
}
// Exists determine whether the file exists
func Exists(name string) bool {
if _, err := os.Stat(name); err != nil {
@ -56,7 +118,7 @@ func CreateTempFile(r io.ReadCloser) (*os.File, error) {
// GetFileType get file type
func GetFileType(filename string) int {
ext := Ext(filename)
ext := strings.ToLower(Ext(filename))
//if SliceContains(conf.TypesMap[conf.OfficeTypes], ext) {
// return conf.OFFICE
//}

View File

@ -3,7 +3,9 @@ package utils
import (
"crypto/md5"
"crypto/sha1"
"encoding/base64"
"encoding/hex"
"strings"
)
func GetSHA1Encode(data string) string {
@ -17,3 +19,20 @@ func GetMD5Encode(data string) string {
h.Write([]byte(data))
return hex.EncodeToString(h.Sum(nil))
}
var DEC = map[string]string{
"-": "+",
"_": "/",
".": "=",
}
func SafeAtob(data string) (string, error) {
for k, v := range DEC {
data = strings.ReplaceAll(data, k, v)
}
bytes, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return "", err
}
return string(bytes), err
}

View File

@ -40,3 +40,32 @@ func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, p
}))
return err
}
type limitWriter struct {
w io.Writer
count int64
limit int64
}
func (l limitWriter) Write(p []byte) (n int, err error) {
wn := int(l.limit - l.count)
if wn > len(p) {
wn = len(p)
}
if wn > 0 {
if n, err = l.w.Write(p[:wn]); err != nil {
return
}
if n < wn {
err = io.ErrShortWrite
}
}
if err == nil {
n = len(p)
}
return
}
func LimitWriter(w io.Writer, size int64) io.Writer {
return &limitWriter{w: w, limit: size}
}

View File

@ -184,7 +184,7 @@ func FsRemove(c *gin.Context) {
return
}
}
fs.ClearCache(req.Dir)
//fs.ClearCache(req.Dir)
common.SuccessResp(c)
}

View File

@ -26,8 +26,9 @@ type ListReq struct {
}
type DirReq struct {
Path string `json:"path" form:"path"`
Password string `json:"password" form:"password"`
Path string `json:"path" form:"path"`
Password string `json:"password" form:"password"`
ForceRoot bool `json:"force_root" form:"force_root"`
}
type ObjResp struct {
@ -93,7 +94,14 @@ func FsDirs(c *gin.Context) {
return
}
user := c.MustGet("user").(*model.User)
req.Path = stdpath.Join(user.BasePath, req.Path)
if req.ForceRoot {
if !user.IsAdmin() {
common.ErrorStrResp(c, "Permission denied", 403)
return
}
} else {
req.Path = stdpath.Join(user.BasePath, req.Path)
}
meta, err := db.GetNearestMeta(req.Path)
if err != nil {
if !errors.Is(errors.Cause(err), errs.MetaNotFound) {
@ -250,27 +258,27 @@ func FsGet(c *gin.Context) {
if err == nil {
provider = storage.Config().Name
}
// file have raw url
if !obj.IsDir() {
if u, ok := obj.(model.URL); ok {
rawURL = u.URL()
} else {
if err != nil {
common.ErrorResp(c, err, 500)
return
if err != nil {
common.ErrorResp(c, err, 500)
return
}
if storage.Config().MustProxy() || storage.GetStorage().WebProxy {
if storage.GetStorage().DownProxyUrl != "" {
rawURL = fmt.Sprintf("%s%s?sign=%s", strings.Split(storage.GetStorage().DownProxyUrl, "\n")[0], req.Path, sign.Sign(obj.GetName()))
} else {
rawURL = fmt.Sprintf("%s/p%s?sign=%s",
common.GetApiUrl(c.Request),
utils.EncodePath(req.Path, true),
sign.Sign(obj.GetName()))
}
if storage.Config().MustProxy() || storage.GetStorage().WebProxy {
if storage.GetStorage().DownProxyUrl != "" {
rawURL = fmt.Sprintf("%s%s?sign=%s", strings.Split(storage.GetStorage().DownProxyUrl, "\n")[0], req.Path, sign.Sign(obj.GetName()))
} else {
rawURL = fmt.Sprintf("%s/p%s?sign=%s",
common.GetApiUrl(c.Request),
utils.EncodePath(req.Path, true),
sign.Sign(obj.GetName()))
}
} else {
// file have raw url
if u, ok := obj.(model.URL); ok {
rawURL = u.URL()
} else {
// if storage is not proxy, use raw url by fs.Link
link, _, err := fs.Link(c, req.Path, model.LinkArgs{IP: c.ClientIP()})
link, _, err := fs.Link(c, req.Path, model.LinkArgs{IP: c.ClientIP(), Header: c.Request.Header})
if err != nil {
common.ErrorResp(c, err, 500)
return

View File

@ -7,9 +7,9 @@ import (
"github.com/alist-org/alist/v3/internal/conf"
"github.com/alist-org/alist/v3/internal/setting"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/alist-org/alist/v3/server/common"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
)
func Favicon(c *gin.Context) {
@ -18,20 +18,33 @@ func Favicon(c *gin.Context) {
func Plist(c *gin.Context) {
link := c.Param("link")
u, err := url.PathUnescape(link)
u, err := utils.SafeAtob(link)
if err != nil {
common.ErrorResp(c, err, 500)
common.ErrorResp(c, err, 400)
return
}
uUrl, err := url.Parse(u)
if err != nil {
common.ErrorResp(c, err, 500)
common.ErrorResp(c, err, 400)
return
}
name := c.Param("name")
log.Debug("name", name)
u = uUrl.String()
name = strings.TrimSuffix(name, ".plist")
fullName := c.Param("name")
Url := uUrl.String()
fullName = strings.TrimSuffix(fullName, ".plist")
fullName, err = utils.SafeAtob(fullName)
if err != nil {
common.ErrorResp(c, err, 400)
return
}
name := fullName
identifier := fmt.Sprintf("ci.nn.%s", url.PathEscape(fullName))
sep := "@"
if strings.Contains(fullName, sep) {
ss := strings.Split(fullName, sep)
name = strings.Join(ss[:len(ss)-1], sep)
identifier = ss[len(ss)-1]
}
name = strings.ReplaceAll(name, "<", "[")
name = strings.ReplaceAll(name, ">", "]")
plist := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
@ -46,13 +59,13 @@ func Plist(c *gin.Context) {
<key>kind</key>
<string>software-package</string>
<key>url</key>
<string>%s</string>
<string><![CDATA[%s]]></string>
</dict>
</array>
<key>metadata</key>
<dict>
<key>bundle-identifier</key>
<string>ci.nn.%s</string>
<string>%s</string>
<key>bundle-version</key>
<string>4.4</string>
<key>kind</key>
@ -63,7 +76,7 @@ func Plist(c *gin.Context) {
</dict>
</array>
</dict>
</plist>`, u, url.PathEscape(name), name)
</plist>`, Url, identifier, name)
c.Header("Content-Type", "application/xml;charset=utf-8")
c.Status(200)
_, _ = c.Writer.WriteString(plist)

View File

@ -271,7 +271,7 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i
if err := fs.Remove(ctx, reqPath); err != nil {
return http.StatusMethodNotAllowed, err
}
fs.ClearCache(path.Dir(reqPath))
//fs.ClearCache(path.Dir(reqPath))
return http.StatusNoContent, nil
}