feat: add Google Photo support (#1853)

* feat: add Google Photo support

* fix: fetch all pages

* chore(google_photo): add meta info

Co-authored-by: Noah Hsu <i@nn.ci>

.github/FUNDING.yml

@@ -0,0 +1,13 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+custom: ['https://alist.nn.ci/guide/sponsor.html']

.github/workflows/issue_close_inactive.yml

@@ -3,6 +3,7 @@ name: Close inactive
 on:
   schedule:
     - cron: "0 0 */7 * *"
+  workflow_dispatch:
 
 jobs:
   close-inactive:

.github/workflows/issue_close_question.yml

@@ -3,6 +3,7 @@ name: Close need info
 on:
   schedule:
     - cron: "0 0 */7 * *"
+  workflow_dispatch:
 
 jobs:
   close-need-info:

README.md

@@ -100,4 +100,4 @@ The `AList` is open-source software licensed under the AGPL-3.0 license.
 
 ---
 
-> [@Blog](https://nn.ci/) · [@GitHub](https://github.com/Xhofe) · [@TelegramGroup](https://t.me/alist_chat) · [@QQGroup](https://jq.qq.com/?_wv=1027&k=YJJj2Gwb)
+> [@Blog](https://nn.ci/) · [@GitHub](https://github.com/Xhofe) · [@TelegramGroup](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2)

README_cn.md

@@ -100,4 +100,4 @@
 
 ---
 
-> [@博客](https://nn.ci/) · [@GitHub](https://github.com/Xhofe) · [@Telegram群](https://t.me/alist_chat) · [@QQ群](https://jq.qq.com/?_wv=1027&k=YJJj2Gwb)
+> [@博客](https://nn.ci/) · [@GitHub](https://github.com/Xhofe) · [@Telegram群](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2)

drivers/all.go

@@ -11,6 +11,7 @@ import (
 	_ "github.com/alist-org/alist/v3/drivers/baidu_photo"
 	_ "github.com/alist-org/alist/v3/drivers/ftp"
 	_ "github.com/alist-org/alist/v3/drivers/google_drive"
+	_ "github.com/alist-org/alist/v3/drivers/google_photo"
 	_ "github.com/alist-org/alist/v3/drivers/lanzou"
 	_ "github.com/alist-org/alist/v3/drivers/local"
 	_ "github.com/alist-org/alist/v3/drivers/mediatrack"

drivers/google_photo/driver.go

@@ -0,0 +1,170 @@
+package google_photo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/alist-org/alist/v3/drivers/base"
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/errs"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	"github.com/go-resty/resty/v2"
+)
+
+type GooglePhoto struct {
+	model.Storage
+	Addition
+	AccessToken string
+}
+
+func (d *GooglePhoto) Config() driver.Config {
+	return config
+}
+
+func (d *GooglePhoto) GetAddition() driver.Additional {
+	return d.Addition
+}
+
+func (d *GooglePhoto) Init(ctx context.Context, storage model.Storage) error {
+	d.Storage = storage
+	err := utils.Json.UnmarshalFromString(d.Storage.Addition, &d.Addition)
+	if err != nil {
+		return err
+	}
+	return d.refreshToken()
+}
+
+func (d *GooglePhoto) Drop(ctx context.Context) error {
+	return nil
+}
+
+func (d *GooglePhoto) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
+	files, err := d.getFiles()
+	if err != nil {
+		return nil, err
+	}
+	return utils.SliceConvert(files, func(src MediaItem) (model.Obj, error) {
+		return fileToObj(src), nil
+	})
+}
+
+//func (d *GooglePhoto) Get(ctx context.Context, path string) (model.Obj, error) {
+//	// this is optional
+//	return nil, errs.NotImplement
+//}
+
+func (d *GooglePhoto) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
+	f, err := d.getFile(file.GetID())
+	if err != nil {
+		return nil, err
+	}
+
+	if strings.Contains(f.MimeType, "image/") {
+		return &model.Link{
+			URL: f.BaseURL + "=d",
+		}, nil
+	} else if strings.Contains(f.MimeType, "video/") {
+		return &model.Link{
+			URL: f.BaseURL + "=dv",
+		}, nil
+	}
+	return &model.Link{}, nil
+}
+
+func (d *GooglePhoto) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
+	return errs.NotSupport
+}
+
+func (d *GooglePhoto) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
+	return errs.NotSupport
+}
+
+func (d *GooglePhoto) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
+	return errs.NotSupport
+}
+
+func (d *GooglePhoto) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
+	return errs.NotSupport
+}
+
+func (d *GooglePhoto) Remove(ctx context.Context, obj model.Obj) error {
+	return errs.NotSupport
+}
+
+func (d *GooglePhoto) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {
+	var e Error
+	// Create resumable upload url
+	postHeaders := map[string]string{
+		"Authorization":              "Bearer " + d.AccessToken,
+		"Content-type":               "application/octet-stream",
+		"X-Goog-Upload-Command":      "start",
+		"X-Goog-Upload-Content-Type": stream.GetMimetype(),
+		"X-Goog-Upload-Protocol":     "resumable",
+		"X-Goog-Upload-Raw-Size":     strconv.FormatInt(stream.GetSize(), 10),
+	}
+	url := "https://photoslibrary.googleapis.com/v1/uploads"
+	res, err := base.NoRedirectClient.R().SetHeaders(postHeaders).
+		SetError(&e).
+		Post(url)
+
+	if err != nil {
+		return err
+	}
+	if e.Error.Code != 0 {
+		if e.Error.Code == 401 {
+			err = d.refreshToken()
+			if err != nil {
+				return err
+			}
+			return d.Put(ctx, dstDir, stream, up)
+		}
+		return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors)
+	}
+
+	//Upload to the Google Photo
+	postUrl := res.Header().Get("X-Goog-Upload-URL")
+	//chunkSize := res.Header().Get("X-Goog-Upload-Chunk-Granularity")
+	postHeaders = map[string]string{
+		"X-Goog-Upload-Command": "upload, finalize",
+		"X-Goog-Upload-Offset":  "0",
+	}
+
+	resp, err := d.request(postUrl, http.MethodPost, func(req *resty.Request) {
+		req.SetBody(stream.GetReadCloser())
+	}, nil, postHeaders)
+
+	if err != nil {
+		return err
+	}
+	//Create MediaItem
+	createItemUrl := "https://photoslibrary.googleapis.com/v1/mediaItems:batchCreate"
+
+	postHeaders = map[string]string{
+		"X-Goog-Upload-Command": "upload, finalize",
+		"X-Goog-Upload-Offset":  "0",
+	}
+
+	data := base.Json{
+		"newMediaItems": []base.Json{
+			{
+				"description": "item-description",
+				"simpleMediaItem": base.Json{
+					"fileName":    stream.GetName(),
+					"uploadToken": string(resp),
+				},
+			},
+		},
+	}
+
+	_, err = d.request(createItemUrl, http.MethodPost, func(req *resty.Request) {
+		req.SetBody(data)
+	}, nil, postHeaders)
+
+	return err
+}
+
+var _ driver.Driver = (*GooglePhoto)(nil)

drivers/google_photo/meta.go

@@ -0,0 +1,29 @@
+package google_photo
+
+import (
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/op"
+)
+
+type Addition struct {
+	driver.RootID
+	RefreshToken string `json:"refresh_token" required:"true"`
+	ClientID     string `json:"client_id" required:"true" default:"202264815644.apps.googleusercontent.com"`
+	ClientSecret string `json:"client_secret" required:"true" default:"X4Z3ca8xfWDb1Voo-F9a7ZxJ"`
+}
+
+var config = driver.Config{
+	Name:        "GooglePhoto",
+	OnlyProxy:   true,
+	DefaultRoot: "root",
+	NoUpload:    true,
+	LocalSort:   true,
+}
+
+func New() driver.Driver {
+	return &GooglePhoto{}
+}
+
+func init() {
+	op.RegisterDriver(config, New)
+}

drivers/google_photo/types.go

@@ -0,0 +1,69 @@
+package google_photo
+
+import (
+	"time"
+
+	"github.com/alist-org/alist/v3/internal/model"
+)
+
+type TokenError struct {
+	Error            string `json:"error"`
+	ErrorDescription string `json:"error_description"`
+}
+
+type Files struct {
+	NextPageToken string      `json:"nextPageToken"`
+	MediaItems    []MediaItem `json:"mediaItems"`
+}
+
+type MediaItem struct {
+	Id            string        `json:"id"`
+	BaseURL       string        `json:"baseUrl"`
+	MimeType      string        `json:"mimeType"`
+	FileName      string        `json:"filename"`
+	MediaMetadata MediaMetadata `json:"mediaMetadata"`
+}
+
+type MediaMetadata struct {
+	CreationTime time.Time `json:"creationTime"`
+	Width        string    `json:"width"`
+	Height       string    `json:"height"`
+	Photo        Photo     `json:"photo,omitempty"`
+	Video        Video     `json:"video,omitempty"`
+}
+
+type Photo struct {
+}
+
+type Video struct {
+}
+
+func fileToObj(f MediaItem) *model.ObjThumb {
+	//size, _ := strconv.ParseInt(f.Size, 10, 64)
+	return &model.ObjThumb{
+		Object: model.Object{
+			ID:       f.Id,
+			Name:     f.FileName,
+			Size:     0,
+			Modified: f.MediaMetadata.CreationTime,
+			IsFolder: false,
+		},
+		Thumbnail: model.Thumbnail{
+			Thumbnail: f.BaseURL + "=w100-h100-c",
+		},
+	}
+}
+
+type Error struct {
+	Error struct {
+		Errors []struct {
+			Domain       string `json:"domain"`
+			Reason       string `json:"reason"`
+			Message      string `json:"message"`
+			LocationType string `json:"location_type"`
+			Location     string `json:"location"`
+		}
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+	} `json:"error"`
+}

drivers/google_photo/util.go

@@ -0,0 +1,105 @@
+package google_photo
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/alist-org/alist/v3/drivers/base"
+	"github.com/go-resty/resty/v2"
+)
+
+// do others that not defined in Driver interface
+
+func (d *GooglePhoto) refreshToken() error {
+	url := "https://www.googleapis.com/oauth2/v4/token"
+	var resp base.TokenResp
+	var e TokenError
+	_, err := base.RestyClient.R().SetResult(&resp).SetError(&e).
+		SetFormData(map[string]string{
+			"client_id":     d.ClientID,
+			"client_secret": d.ClientSecret,
+			"refresh_token": d.RefreshToken,
+			"grant_type":    "refresh_token",
+		}).Post(url)
+	if err != nil {
+		return err
+	}
+	if e.Error != "" {
+		return fmt.Errorf(e.Error)
+	}
+	d.AccessToken = resp.AccessToken
+	return nil
+}
+
+func (d *GooglePhoto) request(url string, method string, callback base.ReqCallback, resp interface{}, headers map[string]string) ([]byte, error) {
+	req := base.RestyClient.R()
+	req.SetHeader("Authorization", "Bearer "+d.AccessToken)
+	if headers != nil {
+		req.SetHeaders(headers)
+	}
+
+	if callback != nil {
+		callback(req)
+	}
+	if resp != nil {
+		req.SetResult(resp)
+	}
+	var e Error
+	req.SetError(&e)
+	res, err := req.Execute(method, url)
+	if err != nil {
+		return nil, err
+	}
+	if e.Error.Code != 0 {
+		if e.Error.Code == 401 {
+			err = d.refreshToken()
+			if err != nil {
+				return nil, err
+			}
+			return d.request(url, method, callback, resp, headers)
+		}
+		return nil, fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors)
+	}
+	return res.Body(), nil
+}
+
+func (d *GooglePhoto) getFiles() ([]MediaItem, error) {
+	pageToken := "first"
+	res := make([]MediaItem, 0)
+	for pageToken != "" {
+		if pageToken == "first" {
+			pageToken = ""
+		}
+		var resp Files
+		query := map[string]string{
+			"fields":    "mediaItems(id,baseUrl,mimeType,mediaMetadata,filename),nextPageToken",
+			"pageSize":  "100",
+			"pageToken": pageToken,
+		}
+		_, err := d.request("https://photoslibrary.googleapis.com/v1/mediaItems", http.MethodGet, func(req *resty.Request) {
+			req.SetQueryParams(query)
+		}, &resp, nil)
+		if err != nil {
+			return nil, err
+		}
+		pageToken = resp.NextPageToken
+		res = append(res, resp.MediaItems...)
+	}
+	return res, nil
+}
+
+func (d *GooglePhoto) getFile(id string) (MediaItem, error) {
+	var resp MediaItem
+
+	query := map[string]string{
+		"fields": "baseUrl,mimeType",
+	}
+	_, err := d.request(fmt.Sprintf("https://photoslibrary.googleapis.com/v1/mediaItems/%s", id), http.MethodGet, func(req *resty.Request) {
+		req.SetQueryParams(query)
+	}, &resp, nil)
+	if err != nil {
+		return resp, err
+	}
+
+	return resp, nil
+}

drivers/onedrive/meta.go

@@ -7,7 +7,7 @@ import (
 
 type Addition struct {
 	driver.RootPath
-	Region       string `json:"region" type:"select" required:"true" options:"global,cn,us,de"`
+	Region       string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"`
 	IsSharepoint bool   `json:"is_sharepoint"`
 	ClientID     string `json:"client_id" required:"true"`
 	ClientSecret string `json:"client_secret" required:"true"`

drivers/onedrive/util.go

@@ -166,7 +166,7 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil
 	}
 	uploadUrl := jsoniter.Get(res, "uploadUrl").ToString()
 	var finish int64 = 0
-	const DEFAULT = 4 * 1024 * 1024
+	const DEFAULT = 100 * 1024 * 1024
 	for finish < stream.GetSize() {
 		if utils.IsCanceled(ctx) {
 			return ctx.Err()

internal/bootstrap/aria2.go

@@ -6,11 +6,9 @@ import (
 )
 
 func InitAria2() {
-	go func() {
 	_, err := aria2.InitClient(2)
 	if err != nil {
 		//utils.Log.Errorf("failed to init aria2 client: %+v", err)
 		utils.Log.Infof("Aria2 not ready.")
 	}
-	}()
 }

internal/bootstrap/config.go

@@ -1,7 +1,6 @@
 package bootstrap
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -25,7 +24,7 @@ func InitConfig() {
 			log.Fatalf("failed to create default config file")
 		}
 	} else {
-		configBytes, err := ioutil.ReadFile(flags.Config)
+		configBytes, err := os.ReadFile(flags.Config)
 		if err != nil {
 			log.Fatalf("reading config file error: %+v", err)
 		}
@@ -39,7 +38,7 @@ func InitConfig() {
 		if err != nil {
 			log.Fatalf("marshal config error: %+v", err)
 		}
-		err = ioutil.WriteFile(flags.Config, confBody, 0777)
+		err = os.WriteFile(flags.Config, confBody, 0777)
 		if err != nil {
 			log.Fatalf("update config struct error: %+v", err)
 		}
@@ -59,7 +58,7 @@ func InitConfig() {
 	if err != nil {
 		log.Errorln("failed delete temp file:", err)
 	}
-	err = os.MkdirAll(conf.Conf.TempDir, 0700)
+	err = os.MkdirAll(conf.Conf.TempDir, 0777)
 	if err != nil {
 		log.Fatalf("create temp dir error: %+v", err)
 	}

internal/conf/config.go

@@ -38,6 +38,7 @@ type Config struct {
 	SiteURL        string    `json:"site_url" env:"SITE_URL"`
 	Cdn            string    `json:"cdn" env:"CDN"`
 	JwtSecret      string    `json:"jwt_secret" env:"JWT_SECRET"`
+	TokenExpiresIn int       `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"`
 	Database       Database  `json:"database"`
 	Scheme         Scheme    `json:"scheme"`
 	TempDir        string    `json:"temp_dir" env:"TEMP_DIR"`
@@ -49,7 +50,7 @@ func DefaultConfig() *Config {
 		Address:        "0.0.0.0",
 		Port:           5244,
 		JwtSecret:      random.String(16),
-		Cdn:       "",
+		TokenExpiresIn: 48,
 		TempDir:        "data/temp",
 		Database: Database{
 			Type:        "sqlite3",
@@ -57,7 +58,6 @@ func DefaultConfig() *Config {
 			TablePrefix: "x_",
 			DBFile:      "data/data.db",
 		},
-		// CaCheExpiration: 30,
 		Log: LogConfig{
 			Enable:     true,
 			Name:       "log/log.log",

internal/op/fs.go

@@ -38,6 +38,13 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
 	}
 	path = utils.StandardizePath(path)
 	log.Debugf("op.List %s", path)
+	key := Key(storage, path)
+	if len(refresh) == 0 || !refresh[0] {
+		if files, ok := listCache.Get(key); ok {
+			log.Debugf("use cache when list %s", path)
+			return files, nil
+		}
+	}
 	dir, err := Get(ctx, storage, path)
 	if err != nil {
 		return nil, errors.WithMessage(err, "failed get dir")
@@ -46,22 +53,14 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li
 	if !dir.IsDir() {
 		return nil, errors.WithStack(errs.NotFolder)
 	}
-	if storage.Config().NoCache {
-		objs, err := storage.List(ctx, dir, args)
-		return objs, errors.WithStack(err)
-	}
-	key := Key(storage, path)
-	if len(refresh) == 0 || !refresh[0] {
-		if files, ok := listCache.Get(key); ok && len(files) > 0 {
-			return files, nil
-		}
-	}
 	objs, err, _ := listG.Do(key, func() ([]model.Obj, error) {
 		files, err := storage.List(ctx, dir, args)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to list objs")
 		}
+		if !storage.Config().NoCache && len(files) > 0 {
 			listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))
+		}
 		return files, nil
 	})
 	return objs, err
@@ -128,6 +127,7 @@ func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, er
 			return f, nil
 		}
 	}
+	log.Debugf("cant find obj with name: %s", name)
 	return nil, errors.WithStack(errs.ObjectNotFound)
 }
 

server/common/auth.go

@@ -3,6 +3,7 @@ package common
 import (
 	"time"
 
+	"github.com/alist-org/alist/v3/internal/conf"
 	"github.com/golang-jwt/jwt/v4"
 	"github.com/pkg/errors"
 )
@@ -18,7 +19,7 @@ func GenerateToken(username string) (tokenString string, err error) {
 	claim := UserClaims{
 		Username: username,
 		RegisteredClaims: jwt.RegisteredClaims{
-			ExpiresAt: jwt.NewNumericDate(time.Now().Add(12 * time.Hour)),
+			ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Duration(conf.Conf.TokenExpiresIn) * time.Hour)),
 			IssuedAt:  jwt.NewNumericDate(time.Now()),
 			NotBefore: jwt.NewNumericDate(time.Now()),
 		}}

server/static/config.go

@@ -1,7 +1,7 @@
 package static
 
 import (
-	stdpath "path"
+	"net/url"
 	"strings"
 
 	"github.com/alist-org/alist/v3/internal/conf"
@@ -16,9 +16,13 @@ type SiteConfig struct {
 }
 
 func getSiteConfig() SiteConfig {
+	u, err := url.Parse(conf.Conf.SiteURL)
+	if err != nil {
+		utils.Log.Fatalf("can't parse site_url: %+v", err)
+	}
 	siteConfig := SiteConfig{
 		ApiURL:   conf.Conf.SiteURL,
-		BasePath: stdpath.Base(conf.Conf.SiteURL),
+		BasePath: u.Path,
 		Cdn:      strings.ReplaceAll(strings.TrimSuffix(conf.Conf.Cdn, "/"), "$version", conf.WebVersion),
 	}
 	// try to get old config

server/static/static.go

@@ -21,11 +21,19 @@ func InitIndex() {
 		log.Fatalf("failed to read index.html: %v", err)
 	}
 	conf.RawIndexHtml = string(index)
+	siteConfig := getSiteConfig()
+	replaceMap := map[string]string{
+		"cdn: undefined":       fmt.Sprintf("cdn: '%s'", siteConfig.Cdn),
+		"base_path: undefined": fmt.Sprintf("base_path: '%s'", siteConfig.BasePath),
+		"api: undefined":       fmt.Sprintf("api: '%s'", siteConfig.ApiURL),
+	}
+	for k, v := range replaceMap {
+		conf.RawIndexHtml = strings.Replace(conf.RawIndexHtml, k, v, 1)
+	}
 	UpdateIndex()
 }
 
 func UpdateIndex() {
-	siteConfig := getSiteConfig()
 	favicon := setting.GetStr(conf.Favicon)
 	title := setting.GetStr(conf.SiteTitle)
 	customizeHead := setting.GetStr(conf.CustomizeHead)
@@ -35,9 +43,6 @@ func UpdateIndex() {
 	replaceMap1 := map[string]string{
 		"https://jsd.nn.ci/gh/alist-org/logo@main/logo.svg": favicon,
 		"Loading...":            title,
-		"cdn: undefined":        fmt.Sprintf("cdn: '%s'", siteConfig.Cdn),
-		"base_path: undefined":  fmt.Sprintf("base_path: '%s'", siteConfig.BasePath),
-		"api: undefined":        fmt.Sprintf("api: '%s'", siteConfig.ApiURL),
 		"main_color: undefined": fmt.Sprintf("main_color: '%s'", mainColor),
 	}
 	for k, v := range replaceMap1 {