feat(drivers): new driver UrlTree (close #3268 in #3933)

* feat(drivers): new driver `urls` (close #3268)

* chore: rename

* support customize basic info or get from url

* dfs tree to calculate folder size

* go mod tidy

* add help message

drivers/all.go

@@ -36,6 +36,7 @@ import (
 	_ "github.com/alist-org/alist/v3/drivers/terabox"
 	_ "github.com/alist-org/alist/v3/drivers/thunder"
 	_ "github.com/alist-org/alist/v3/drivers/trainbit"
+	_ "github.com/alist-org/alist/v3/drivers/url_tree"
 	_ "github.com/alist-org/alist/v3/drivers/uss"
 	_ "github.com/alist-org/alist/v3/drivers/virtual"
 	_ "github.com/alist-org/alist/v3/drivers/webdav"

drivers/url_tree/driver.go

@@ -0,0 +1,79 @@
+package url_tree
+
+import (
+	"context"
+	stdpath "path"
+
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/errs"
+	"github.com/alist-org/alist/v3/internal/model"
+	"github.com/alist-org/alist/v3/pkg/utils"
+	log "github.com/sirupsen/logrus"
+)
+
+type Urls struct {
+	model.Storage
+	Addition
+	root *Node
+}
+
+func (d *Urls) Config() driver.Config {
+	return config
+}
+
+func (d *Urls) GetAddition() driver.Additional {
+	return &d.Addition
+}
+
+func (d *Urls) Init(ctx context.Context) error {
+	node, err := BuildTree(d.UrlStructure, d.HeadSize)
+	if err != nil {
+		return err
+	}
+	node.calSize()
+	d.root = node
+	return nil
+}
+
+func (d *Urls) Drop(ctx context.Context) error {
+	return nil
+}
+
+func (d *Urls) Get(ctx context.Context, path string) (model.Obj, error) {
+	node := GetNodeFromRootByPath(d.root, path)
+	return nodeToObj(node, path)
+}
+
+func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
+	node := GetNodeFromRootByPath(d.root, dir.GetPath())
+	log.Debugf("path: %s, node: %+v", dir.GetPath(), node)
+	if node == nil {
+		return nil, errs.ObjectNotFound
+	}
+	if node.isFile() {
+		return nil, errs.NotFolder
+	}
+	return utils.SliceConvert(node.Children, func(node *Node) (model.Obj, error) {
+		return nodeToObj(node, stdpath.Join(dir.GetPath(), node.Name))
+	})
+}
+
+func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
+	node := GetNodeFromRootByPath(d.root, file.GetPath())
+	log.Debugf("path: %s, node: %+v", file.GetPath(), node)
+	if node == nil {
+		return nil, errs.ObjectNotFound
+	}
+	if node.isFile() {
+		return &model.Link{
+			URL: node.Url,
+		}, nil
+	}
+	return nil, errs.NotFile
+}
+
+//func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) {
+//	return nil, errs.NotSupport
+//}
+
+var _ driver.Driver = (*Urls)(nil)

drivers/url_tree/meta.go

@@ -0,0 +1,35 @@
+package url_tree
+
+import (
+	"github.com/alist-org/alist/v3/internal/driver"
+	"github.com/alist-org/alist/v3/internal/op"
+)
+
+type Addition struct {
+	// Usually one of two
+	// driver.RootPath
+	// driver.RootID
+	// define other
+	UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://jsd.nn.ci/gh/alist-org/alist/README.md\nhttps://jsd.nn.ci/gh/alist-org/alist/README_cn.md\nfolder:\n  CONTRIBUTING.md:1635:https://jsd.nn.ci/gh/alist-org/alist/CONTRIBUTING.md\n  CODE_OF_CONDUCT.md:2093:https://jsd.nn.ci/gh/alist-org/alist/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n  [FileName:][FileSize:][Modified:]Url"`
+	HeadSize     bool   `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."`
+}
+
+var config = driver.Config{
+	Name:              "UrlTree",
+	LocalSort:         true,
+	OnlyLocal:         false,
+	OnlyProxy:         false,
+	NoCache:           true,
+	NoUpload:          true,
+	NeedMs:            false,
+	DefaultRoot:       "",
+	CheckStatus:       true,
+	Alert:             "",
+	NoOverwriteUpload: false,
+}
+
+func init() {
+	op.RegisterDriver(func() driver.Driver {
+		return &Urls{}
+	})
+}

drivers/url_tree/types.go

@@ -0,0 +1,46 @@
+package url_tree
+
+// Node is a node in the folder tree
+type Node struct {
+	Url      string
+	Name     string
+	Level    int
+	Modified int64
+	Size     int64
+	Children []*Node
+}
+
+func (node *Node) getByPath(paths []string) *Node {
+	if len(paths) == 0 || node == nil {
+		return nil
+	}
+	if node.Name != paths[0] {
+		return nil
+	}
+	if len(paths) == 1 {
+		return node
+	}
+	for _, child := range node.Children {
+		tmp := child.getByPath(paths[1:])
+		if tmp != nil {
+			return tmp
+		}
+	}
+	return nil
+}
+
+func (node *Node) isFile() bool {
+	return node.Url != ""
+}
+
+func (node *Node) calSize() int64 {
+	if node.isFile() {
+		return node.Size
+	}
+	var size int64 = 0
+	for _, child := range node.Children {
+		size += child.calSize()
+	}
+	node.Size = size
+	return size
+}

drivers/url_tree/urls_test.go

@@ -0,0 +1,47 @@
+package url_tree_test
+
+import (
+	"testing"
+
+	"github.com/alist-org/alist/v3/drivers/url_tree"
+)
+
+func testTree() (*url_tree.Node, error) {
+	text := `folder1:
+  name1:url1
+  url2
+  folder2:
+    url3
+    url4
+  url5
+folder3:
+  url6
+  url7
+url8`
+	return url_tree.BuildTree(text, false)
+}
+
+func TestBuildTree(t *testing.T) {
+	node, err := testTree()
+	if err != nil {
+		t.Errorf("failed to build tree: %+v", err)
+	} else {
+		t.Logf("tree: %+v", node)
+	}
+}
+
+func TestGetNode(t *testing.T) {
+	root, err := testTree()
+	if err != nil {
+		t.Errorf("failed to build tree: %+v", err)
+		return
+	}
+	node := url_tree.GetNodeFromRootByPath(root, "/")
+	if node != root {
+		t.Errorf("got wrong node: %+v", node)
+	}
+	url3 := url_tree.GetNodeFromRootByPath(root, "/folder1/folder2/url3")
+	if url3 != root.Children[0].Children[2].Children[0] {
+		t.Errorf("got wrong node: %+v", url3)
+	}
+}

drivers/url_tree/util.go

@@ -0,0 +1,192 @@
+package url_tree
+
+import (
+	"fmt"
+	stdpath "path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/alist-org/alist/v3/drivers/base"
+	"github.com/alist-org/alist/v3/internal/errs"
+	"github.com/alist-org/alist/v3/internal/model"
+	log "github.com/sirupsen/logrus"
+)
+
+// build tree from text, text structure definition:
+/**
+ * FolderName:
+ *   [FileName:][FileSize:][Modified:]Url
+ */
+/**
+ * For example:
+ * folder1:
+ *   name1:url1
+ *   url2
+ *   folder2:
+ *     url3
+ *     url4
+ *   url5
+ * folder3:
+ *   url6
+ *   url7
+ * url8
+ */
+// if there are no name, use the last segment of url as name
+func BuildTree(text string, headSize bool) (*Node, error) {
+	lines := strings.Split(text, "\n")
+	var root = &Node{Level: -1, Name: "root"}
+	stack := []*Node{root}
+	for _, line := range lines {
+		// calculate indent
+		indent := 0
+		for i := 0; i < len(line); i++ {
+			if line[i] != ' ' {
+				break
+			}
+			indent++
+		}
+		// if indent is not a multiple of 2, it is an error
+		if indent%2 != 0 {
+			return nil, fmt.Errorf("the line '%s' is not a multiple of 2", line)
+		}
+		// calculate level
+		level := indent / 2
+		line = strings.TrimSpace(line[indent:])
+		// if the line is empty, skip
+		if line == "" {
+			continue
+		}
+		// if level isn't greater than the level of the top of the stack
+		// it is not the child of the top of the stack
+		if level <= stack[len(stack)-1].Level {
+			// pop the top of the stack
+			stack = stack[:len(stack)-1]
+		}
+		// if the line is a folder
+		if isFolder(line) {
+			// create a new node
+			node := &Node{
+				Level: level,
+				Name:  strings.TrimSuffix(line, ":"),
+			}
+			// add the node to the top of the stack
+			stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node)
+			// push the node to the stack
+			stack = append(stack, node)
+		} else {
+			// if the line is a file
+			// create a new node
+			node, err := parseFileLine(line, headSize)
+			if err != nil {
+				return nil, err
+			}
+			node.Level = level
+			// add the node to the top of the stack
+			stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node)
+		}
+	}
+	return root, nil
+}
+
+func isFolder(line string) bool {
+	return strings.HasSuffix(line, ":")
+}
+
+// line definition:
+// [FileName:][FileSize:][Modified:]Url
+func parseFileLine(line string, headSize bool) (*Node, error) {
+	// if there is no url, it is an error
+	if !strings.Contains(line, "http://") && !strings.Contains(line, "https://") {
+		return nil, fmt.Errorf("invalid line: %s, because url is required for file", line)
+	}
+	index := strings.Index(line, "http://")
+	if index == -1 {
+		index = strings.Index(line, "https://")
+	}
+	url := line[index:]
+	info := line[:index]
+	node := &Node{
+		Url: url,
+	}
+	haveSize := false
+	if index > 0 {
+		if !strings.HasSuffix(info, ":") {
+			return nil, fmt.Errorf("invalid line: %s, because file info must end with ':'", line)
+		}
+		info = info[:len(info)-1]
+		if info == "" {
+			return nil, fmt.Errorf("invalid line: %s, because file name can't be empty", line)
+		}
+		infoParts := strings.Split(info, ":")
+		node.Name = infoParts[0]
+		if len(infoParts) > 1 {
+			size, err := strconv.ParseInt(infoParts[1], 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("invalid line: %s, because file size must be an integer", line)
+			}
+			node.Size = size
+			haveSize = true
+			if len(infoParts) > 2 {
+				modified, err := strconv.ParseInt(infoParts[2], 10, 64)
+				if err != nil {
+					return nil, fmt.Errorf("invalid line: %s, because file modified must be an unix timestamp", line)
+				}
+				node.Modified = modified
+			}
+		}
+	} else {
+		node.Name = stdpath.Base(url)
+	}
+	if !haveSize && headSize {
+		size, err := getSizeFromUrl(url)
+		if err != nil {
+			log.Errorf("get size from url error: %s", err)
+		} else {
+			node.Size = size
+		}
+	}
+	return node, nil
+}
+
+func splitPath(path string) []string {
+	if path == "/" {
+		return []string{"root"}
+	}
+	parts := strings.Split(path, "/")
+	parts[0] = "root"
+	return parts
+}
+
+func GetNodeFromRootByPath(root *Node, path string) *Node {
+	return root.getByPath(splitPath(path))
+}
+
+func nodeToObj(node *Node, path string) (model.Obj, error) {
+	if node == nil {
+		return nil, errs.ObjectNotFound
+	}
+	return &model.Object{
+		Name:     node.Name,
+		Size:     node.Size,
+		Modified: time.Unix(node.Modified, 0),
+		IsFolder: !node.isFile(),
+		Path:     path,
+	}, nil
+}
+
+func getSizeFromUrl(url string) (int64, error) {
+	res, err := base.RestyClient.R().SetDoNotParseResponse(true).Head(url)
+	if err != nil {
+		return 0, err
+	}
+	defer res.RawResponse.Body.Close()
+	if res.StatusCode() >= 300 {
+		return 0, fmt.Errorf("get size from url %s failed, status code: %d", url, res.StatusCode())
+	}
+	size, err := strconv.ParseInt(res.Header().Get("Content-Length"), 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return size, nil
+}