Compare commits

..

7 Commits

Author SHA1 Message Date
b8cf02ca68 fix(aria2): retry 5 times for get status (close #1857) 2022-10-18 15:27:19 +08:00
3db798a82a feat(google_photo): Add categories in root, add album support. (#2046)
* feat(google_photo): Add categories in root, add album support.

* fix(google_photo): Remove else block in `drive/google_photo/types.go:60`
2022-10-18 15:19:05 +08:00
45cc0cedbd fix(s3): mkdir and delete (close #2029) 2022-10-18 15:10:47 +08:00
2efade123e fix(189pc):slice bounds out of range close #2045 2022-10-17 22:39:51 +08:00
fc393f743f fix(thunder):no additional processing when the deviceID is correct 2022-10-17 22:37:17 +08:00
0e99e7e9b9 fix(thunder,189pc): some known problems 2022-10-17 00:54:39 +08:00
7a95850c1b fix(google_drive): incorrect ModifiedTime (close #2002) 2022-10-14 14:17:33 +08:00
12 changed files with 212 additions and 92 deletions

View File

@ -110,7 +110,9 @@ func ParseHttpHeader(str string) map[string]string {
header := make(map[string]string)
for _, value := range strings.Split(str, "&") {
i := strings.Index(value, "=")
header[strings.TrimSpace(value[0:i])] = strings.TrimSpace(value[i+1:])
if i > 0 {
header[strings.TrimSpace(value[0:i])] = strings.TrimSpace(value[i+1:])
}
}
return header
}

View File

@ -15,6 +15,7 @@ type Addition struct {
Type string `json:"type" type:"select" options:"personal,family" default:"personal"`
FamilyID string `json:"family_id"`
RapidUpload bool `json:"rapid_upload"`
NonuseOrc bool `json:"nonuse_orc"`
}
var config = driver.Config{

View File

@ -186,20 +186,19 @@ func (y *Yun189PC) getFiles(ctx context.Context, fileId string) ([]model.Obj, er
func (y *Yun189PC) login() (err error) {
// 初始化登陆所需参数
if y.loginParam == nil {
if y.loginParam == nil || !y.NonuseOrc {
if err = y.initLoginParam(); err != nil {
// 验证码也通过错误返回
return err
}
}
defer func() {
// 销毁验证码
y.VCode = ""
// 销毁登陆参数
y.loginParam = nil
// 遇到错误,重新加载登陆参数
if err != nil {
if err != nil && y.NonuseOrc {
if err1 := y.initLoginParam(); err1 != nil {
err = fmt.Errorf("err1: %s \nerr2: %s", err, err1)
}
@ -303,44 +302,34 @@ func (y *Yun189PC) initLoginParam() error {
param.jRsaKey = fmt.Sprintf("-----BEGIN PUBLIC KEY-----\n%s\n-----END PUBLIC KEY-----", encryptConf.Data.PubKey)
param.RsaUsername = encryptConf.Data.Pre + RsaEncrypt(param.jRsaKey, y.Username)
param.RsaPassword = encryptConf.Data.Pre + RsaEncrypt(param.jRsaKey, y.Password)
// 判断是否需要验证码
res, err = y.client.R().
SetFormData(map[string]string{
"appKey": APP_ID,
"accountType": ACCOUNT_TYPE,
"userName": param.RsaUsername,
}).
Post(AUTH_URL + "/api/logbox/oauth2/needcaptcha.do")
if err != nil {
return err
}
y.loginParam = &param
if res.String() != "0" {
imgRes, err := y.client.R().
SetQueryParams(map[string]string{
"token": param.CaptchaToken,
"REQID": param.ReqId,
"rnd": fmt.Sprint(timestamp()),
}).
Get(AUTH_URL + "/api/logbox/oauth2/picCaptcha.do")
if err != nil {
return fmt.Errorf("failed to obtain verification code")
imgRes, err := y.client.R().
SetQueryParams(map[string]string{
"token": param.CaptchaToken,
"REQID": param.ReqId,
"rnd": fmt.Sprint(timestamp()),
}).
Get(AUTH_URL + "/api/logbox/oauth2/picCaptcha.do")
if err != nil {
return fmt.Errorf("failed to obtain verification code")
}
if imgRes.Size() > 20 {
if setting.GetStr(conf.OcrApi) != "" && !y.NonuseOrc {
vRes, err := base.RestyClient.R().
SetMultipartField("image", "validateCode.png", "image/png", bytes.NewReader(imgRes.Body())).
Post(setting.GetStr(conf.OcrApi))
if err != nil {
return err
}
if jsoniter.Get(vRes.Body(), "status").ToInt() == 200 {
y.VCode = jsoniter.Get(vRes.Body(), "result").ToString()
return nil
}
}
// 尝试使用ocr
vRes, err := base.RestyClient.R().
SetMultipartField("image", "validateCode.png", "image/png", bytes.NewReader(imgRes.Body())).
Post(setting.GetStr(conf.OcrApi))
if err == nil && jsoniter.Get(vRes.Body(), "status").ToInt() == 200 {
y.VCode = jsoniter.Get(vRes.Body(), "result").ToString()
}
// ocr无法处理返回验证码图片给前端
if len(y.VCode) != 4 {
return fmt.Errorf("need validate code: data:image/png;base64,%s", base64.StdEncoding.EncodeToString(res.Body()))
}
// 返回验证码图片给前端
return fmt.Errorf(`need img validate code: <img src="data:image/png;base64,%s"/>`, base64.StdEncoding.EncodeToString(imgRes.Body()))
}
return nil
}
@ -396,6 +385,7 @@ func (y *Yun189PC) CommonUpload(ctx context.Context, dstDir model.Obj, file mode
const DEFAULT int64 = 10485760
var count = int64(math.Ceil(float64(file.GetSize()) / float64(DEFAULT)))
requestID := uuid.NewString()
params := Params{
"parentFolderId": dstDir.GetID(),
"fileName": url.QueryEscape(file.GetName()),
@ -417,6 +407,7 @@ func (y *Yun189PC) CommonUpload(ctx context.Context, dstDir model.Obj, file mode
var initMultiUpload InitMultiUploadResp
_, err = y.request(fullUrl+"/initMultiUpload", http.MethodGet, func(req *resty.Request) {
req.SetContext(ctx)
req.SetHeader("X-Request-ID", requestID)
}, params, &initMultiUpload)
if err != nil {
return err
@ -451,6 +442,7 @@ func (y *Yun189PC) CommonUpload(ctx context.Context, dstDir model.Obj, file mode
_, err = y.request(fullUrl+"/getMultiUploadUrls", http.MethodGet,
func(req *resty.Request) {
req.SetContext(ctx)
req.SetHeader("X-Request-ID", requestID)
}, Params{
"partInfo": fmt.Sprintf("%d-%s", i, silceMd5Base64),
"uploadFileId": initMultiUpload.Data.UploadFileID,
@ -486,6 +478,7 @@ func (y *Yun189PC) CommonUpload(ctx context.Context, dstDir model.Obj, file mode
_, err = y.request(fullUrl+"/commitMultiUploadFile", http.MethodGet,
func(req *resty.Request) {
req.SetContext(ctx)
req.SetHeader("X-Request-ID", requestID)
}, Params{
"uploadFileId": initMultiUpload.Data.UploadFileID,
"fileMd5": fileMd5Hex,
@ -542,6 +535,7 @@ func (y *Yun189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.
sliceMd5Hex = strings.ToUpper(utils.GetMD5Encode(strings.Join(silceMd5Hexs, "\n")))
}
requestID := uuid.NewString()
// 检测是否支持快传
params := Params{
"parentFolderId": dstDir.GetID(),
@ -564,6 +558,7 @@ func (y *Yun189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.
var uploadInfo InitMultiUploadResp
_, err = y.request(fullUrl+"/initMultiUpload", http.MethodGet, func(req *resty.Request) {
req.SetContext(ctx)
req.SetHeader("X-Request-ID", requestID)
}, params, &uploadInfo)
if err != nil {
return err
@ -575,6 +570,7 @@ func (y *Yun189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.
_, err = y.request(fullUrl+"/getMultiUploadUrls", http.MethodGet,
func(req *resty.Request) {
req.SetContext(ctx)
req.SetHeader("X-Request-ID", requestID)
}, Params{
"uploadFileId": uploadInfo.Data.UploadFileID,
"partInfo": strings.Join(silceMd5Base64s, ","),
@ -611,6 +607,7 @@ func (y *Yun189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.
_, err = y.request(fullUrl+"/commitMultiUploadFile", http.MethodGet,
func(req *resty.Request) {
req.SetContext(ctx)
req.SetHeader("X-Request-ID", requestID)
}, Params{
"uploadFileId": uploadInfo.Data.UploadFileID,
"isLog": "0",

View File

@ -33,7 +33,7 @@ func fileToObj(f File) *model.ObjThumb {
ID: f.Id,
Name: f.Name,
Size: size,
Modified: time.Time{},
Modified: f.ModifiedTime,
IsFolder: f.MimeType == "application/vnd.google-apps.folder",
},
Thumbnail: model.Thumbnail{},

View File

@ -43,7 +43,7 @@ func (d *GooglePhoto) Drop(ctx context.Context) error {
}
func (d *GooglePhoto) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
files, err := d.getFiles()
files, err := d.getFiles(dir.GetID())
if err != nil {
return nil, err
}
@ -58,7 +58,7 @@ func (d *GooglePhoto) List(ctx context.Context, dir model.Obj, args model.ListAr
//}
func (d *GooglePhoto) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
f, err := d.getFile(file.GetID())
f, err := d.getMedia(file.GetID())
if err != nil {
return nil, err
}

View File

@ -10,6 +10,7 @@ type Addition struct {
RefreshToken string `json:"refresh_token" required:"true"`
ClientID string `json:"client_id" required:"true" default:"202264815644.apps.googleusercontent.com"`
ClientSecret string `json:"client_secret" required:"true" default:"X4Z3ca8xfWDb1Voo-F9a7ZxJ"`
ShowArchive bool `json:"show_archive"`
}
var config = driver.Config{

View File

@ -1,6 +1,7 @@
package google_photo
import (
"reflect"
"time"
"github.com/alist-org/alist/v3/internal/model"
@ -11,17 +12,21 @@ type TokenError struct {
ErrorDescription string `json:"error_description"`
}
type Files struct {
type Items struct {
NextPageToken string `json:"nextPageToken"`
MediaItems []MediaItem `json:"mediaItems"`
MediaItems []MediaItem `json:"mediaItems,omitempty"`
Albums []MediaItem `json:"albums,omitempty"`
SharedAlbums []MediaItem `json:"sharedAlbums,omitempty"`
}
type MediaItem struct {
Id string `json:"id"`
BaseURL string `json:"baseUrl"`
MimeType string `json:"mimeType"`
FileName string `json:"filename"`
MediaMetadata MediaMetadata `json:"mediaMetadata"`
Id string `json:"id"`
Title string `json:"title,omitempty"`
BaseURL string `json:"baseUrl,omitempty"`
CoverPhotoBaseUrl string `json:"coverPhotoBaseUrl,omitempty"`
MimeType string `json:"mimeType,omitempty"`
FileName string `json:"filename,omitempty"`
MediaMetadata MediaMetadata `json:"mediaMetadata,omitempty"`
}
type MediaMetadata struct {
@ -39,18 +44,29 @@ type Video struct {
}
func fileToObj(f MediaItem) *model.ObjThumb {
//size, _ := strconv.ParseInt(f.Size, 10, 64)
if !reflect.DeepEqual(f.MediaMetadata, MediaMetadata{}){
return &model.ObjThumb{
Object: model.Object{
ID: f.Id,
Name: f.FileName,
Size: 0,
Modified: f.MediaMetadata.CreationTime,
IsFolder: false,
},
Thumbnail: model.Thumbnail{
Thumbnail: f.BaseURL + "=w100-h100-c",
},
}
}
return &model.ObjThumb{
Object: model.Object{
ID: f.Id,
Name: f.FileName,
Name: f.Title,
Size: 0,
Modified: f.MediaMetadata.CreationTime,
IsFolder: false,
},
Thumbnail: model.Thumbnail{
Thumbnail: f.BaseURL + "=w100-h100-c",
Modified: time.Time{},
IsFolder: true,
},
Thumbnail: model.Thumbnail{},
}
}

View File

@ -10,6 +10,13 @@ import (
// do others that not defined in Driver interface
const (
FETCH_ALL = "all"
FETCH_ALBUMS = "albums"
FETCH_ROOT = "root"
FETCH_SHARE_ALBUMS = "share_albums"
)
func (d *GooglePhoto) refreshToken() error {
url := "https://www.googleapis.com/oauth2/v4/token"
var resp base.TokenResp
@ -34,6 +41,7 @@ func (d *GooglePhoto) refreshToken() error {
func (d *GooglePhoto) request(url string, method string, callback base.ReqCallback, resp interface{}, headers map[string]string) ([]byte, error) {
req := base.RestyClient.R()
req.SetHeader("Authorization", "Bearer "+d.AccessToken)
req.SetHeader("Accept-Encoding", "gzip")
if headers != nil {
req.SetHeaders(headers)
}
@ -63,32 +71,83 @@ func (d *GooglePhoto) request(url string, method string, callback base.ReqCallba
return res.Body(), nil
}
func (d *GooglePhoto) getFiles() ([]MediaItem, error) {
pageToken := "first"
res := make([]MediaItem, 0)
for pageToken != "" {
if pageToken == "first" {
pageToken = ""
}
var resp Files
query := map[string]string{
"fields": "mediaItems(id,baseUrl,mimeType,mediaMetadata,filename),nextPageToken",
"pageSize": "100",
"pageToken": pageToken,
}
_, err := d.request("https://photoslibrary.googleapis.com/v1/mediaItems", http.MethodGet, func(req *resty.Request) {
req.SetQueryParams(query)
}, &resp, nil)
if err != nil {
return nil, err
}
pageToken = resp.NextPageToken
res = append(res, resp.MediaItems...)
func (d *GooglePhoto) getFiles(id string) ([]MediaItem, error) {
switch id {
case FETCH_ALL:
return d.getAllMedias()
case FETCH_ALBUMS:
return d.getAlbums()
case FETCH_SHARE_ALBUMS:
return d.getShareAlbums()
case FETCH_ROOT:
return d.getFakeRoot()
default:
return d.getMedias(id)
}
return res, nil
}
func (d *GooglePhoto) getFile(id string) (MediaItem, error) {
func (d *GooglePhoto) getFakeRoot() ([]MediaItem, error) {
return []MediaItem{
{
Id: FETCH_ALL,
Title: "全部媒体",
},
{
Id: FETCH_ALBUMS,
Title: "全部影集",
},
{
Id: FETCH_SHARE_ALBUMS,
Title: "共享影集",
},
}, nil
}
func (d *GooglePhoto) getAlbums() ([]MediaItem, error) {
return d.fetchItems(
"https://photoslibrary.googleapis.com/v1/albums",
map[string]string{
"fields": "albums(id,title,coverPhotoBaseUrl),nextPageToken",
"pageSize": "50",
"pageToken": "first",
},
http.MethodGet)
}
func (d *GooglePhoto) getShareAlbums() ([]MediaItem, error) {
return d.fetchItems(
"https://photoslibrary.googleapis.com/v1/sharedAlbums",
map[string]string{
"fields": "sharedAlbums(id,title,coverPhotoBaseUrl),nextPageToken",
"pageSize": "50",
"pageToken": "first",
},
http.MethodGet)
}
func (d *GooglePhoto) getMedias(albumId string) ([]MediaItem, error) {
return d.fetchItems(
"https://photoslibrary.googleapis.com/v1/mediaItems:search",
map[string]string{
"fields": "mediaItems(id,baseUrl,mimeType,mediaMetadata,filename),nextPageToken",
"pageSize": "100",
"albumId": albumId,
"pageToken": "first",
}, http.MethodPost)
}
func (d *GooglePhoto) getAllMedias() ([]MediaItem, error) {
return d.fetchItems(
"https://photoslibrary.googleapis.com/v1/mediaItems",
map[string]string{
"fields": "mediaItems(id,baseUrl,mimeType,mediaMetadata,filename),nextPageToken",
"pageSize": "100",
"pageToken": "first",
},
http.MethodGet)
}
func (d *GooglePhoto) getMedia(id string) (MediaItem, error) {
var resp MediaItem
query := map[string]string{
@ -103,3 +162,25 @@ func (d *GooglePhoto) getFile(id string) (MediaItem, error) {
return resp, nil
}
func (d *GooglePhoto) fetchItems(url string, query map[string]string, method string) ([]MediaItem, error){
res := make([]MediaItem, 0)
for query["pageToken"] != "" {
if query["pageToken"] == "first" {
query["pageToken"] = ""
}
var resp Items
_, err := d.request(url, method, func(req *resty.Request) {
req.SetQueryParams(query)
}, &resp, nil)
if err != nil {
return nil, err
}
query["pageToken"] = resp.NextPageToken
res = append(res, resp.MediaItems...)
res = append(res, resp.Albums...)
res = append(res, resp.SharedAlbums...)
}
return res, nil
}

View File

@ -52,11 +52,12 @@ func getKey(path string, dir bool) string {
return path
}
// var defaultPlaceholderName = ".placeholder"
var defaultPlaceholderName = ".alist"
func getPlaceholderName(placeholder string) string {
//if placeholder == "" {
// return defaultPlaceholderName
//}
if placeholder == "" {
return defaultPlaceholderName
}
return placeholder
}
@ -88,7 +89,7 @@ func (d *S3) listV1(prefix string) ([]model.Obj, error) {
}
for _, object := range listObjectsResult.Contents {
name := path.Base(*object.Key)
if name == getPlaceholderName(d.Placeholder) {
if name == getPlaceholderName(d.Placeholder) || name == d.Placeholder {
continue
}
file := model.Object{
@ -140,7 +141,7 @@ func (d *S3) listV2(prefix string) ([]model.Obj, error) {
}
for _, object := range listObjectsResult.Contents {
name := path.Base(*object.Key)
if name == getPlaceholderName(d.Placeholder) {
if name == getPlaceholderName(d.Placeholder) || name == d.Placeholder {
continue
}
file := model.Object{
@ -222,11 +223,12 @@ func (d *S3) removeDir(ctx context.Context, src string) error {
}
}
_ = d.removeFile(path.Join(src, getPlaceholderName(d.Placeholder)))
_ = d.removeFile(path.Join(src, d.Placeholder))
return nil
}
func (d *S3) removeFile(src string) error {
key := getKey(src, true)
key := getKey(src, false)
input := &s3.DeleteObjectInput{
Bucket: &d.Bucket,
Key: &key,

View File

@ -59,13 +59,18 @@ func (x *Thunder) Init(ctx context.Context, storage model.Storage) (err error) {
"j",
"4scKJNdd7F27Hv7tbt",
},
DeviceID: "9aa5c268e7bcfc197a9ad88e2fb330e5",
DeviceID: utils.GetMD5Encode(x.Username + x.Password),
ClientID: "Xp6vsxz_7IYVw2BB",
ClientSecret: "Xp6vsy4tN9toTVdMSpomVdXpRmES",
ClientVersion: "7.51.0.8196",
PackageName: "com.xunlei.downloadprovider",
UserAgent: "ANDROID-com.xunlei.downloadprovider/7.51.0.8196 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/220200 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)",
DownloadUserAgent: "Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)",
refreshCTokenCk: func(token string) {
x.CaptchaToken = token
op.MustSaveDriverStorage(x)
},
},
refreshTokenFunc: func() error {
// 通过RefreshToken刷新
@ -88,7 +93,6 @@ func (x *Thunder) Init(ctx context.Context, storage model.Storage) (err error) {
ctoekn := strings.TrimSpace(x.CaptchaToken)
if ctoekn != "" {
x.SetCaptchaToken(ctoekn)
x.CaptchaToken = ""
}
// 防止重复登录
@ -139,7 +143,12 @@ func (x *ThunderExpert) Init(ctx context.Context, storage model.Storage) (err er
Common: &Common{
client: base.NewRestyClient(),
DeviceID: x.DeviceID,
DeviceID: func() string {
if len(x.DeviceID) != 32 {
return utils.GetMD5Encode(x.DeviceID)
}
return x.DeviceID
}(),
ClientID: x.ClientID,
ClientSecret: x.ClientSecret,
ClientVersion: x.ClientVersion,
@ -147,12 +156,16 @@ func (x *ThunderExpert) Init(ctx context.Context, storage model.Storage) (err er
UserAgent: x.UserAgent,
DownloadUserAgent: x.DownloadUserAgent,
UseVideoUrl: x.UseVideoUrl,
refreshCTokenCk: func(token string) {
x.CaptchaToken = token
op.MustSaveDriverStorage(x)
},
},
}
if x.CaptchaToken != "" {
x.SetCaptchaToken(x.CaptchaToken)
x.CaptchaToken = ""
}
// 签名方法
@ -206,7 +219,6 @@ func (x *ThunderExpert) Init(ctx context.Context, storage model.Storage) (err er
// 仅修改验证码token
if x.CaptchaToken != "" {
x.SetCaptchaToken(x.CaptchaToken)
x.CaptchaToken = ""
}
x.XunLeiCommon.UserAgent = x.UserAgent
x.XunLeiCommon.DownloadUserAgent = x.DownloadUserAgent

View File

@ -53,6 +53,9 @@ type Common struct {
UserAgent string
DownloadUserAgent string
UseVideoUrl bool
// 验证码token刷新成功回调
refreshCTokenCk func(token string)
}
func (c *Common) SetCaptchaToken(captchaToken string) {
@ -125,13 +128,16 @@ func (c *Common) refreshCaptchaToken(action string, metas map[string]string) err
}
if resp.Url != "" {
return fmt.Errorf("need verify:%s", resp.Url)
return fmt.Errorf(`need verify: <a target="_blank" href="%s">Click Here</a>`, resp.Url)
}
if resp.CaptchaToken == "" {
return fmt.Errorf("empty captchaToken")
}
if c.refreshCTokenCk != nil {
c.refreshCTokenCk(resp.CaptchaToken)
}
c.SetCaptchaToken(resp.CaptchaToken)
return nil
}

View File

@ -70,6 +70,8 @@ func (m *Monitor) Update() (bool, error) {
info, err := client.TellStatus(m.tsk.ID)
if err != nil {
m.retried++
log.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried)
return false, nil
}
if m.retried > 5 {
return true, errors.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried)