diff --git a/drivers/all.go b/drivers/all.go index 9304853a..28d25f54 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -36,6 +36,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/terabox" _ "github.com/alist-org/alist/v3/drivers/thunder" _ "github.com/alist-org/alist/v3/drivers/trainbit" + _ "github.com/alist-org/alist/v3/drivers/url_tree" _ "github.com/alist-org/alist/v3/drivers/uss" _ "github.com/alist-org/alist/v3/drivers/virtual" _ "github.com/alist-org/alist/v3/drivers/webdav" diff --git a/drivers/url_tree/driver.go b/drivers/url_tree/driver.go new file mode 100644 index 00000000..6a45bb7d --- /dev/null +++ b/drivers/url_tree/driver.go @@ -0,0 +1,79 @@ +package url_tree + +import ( + "context" + stdpath "path" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + log "github.com/sirupsen/logrus" +) + +type Urls struct { + model.Storage + Addition + root *Node +} + +func (d *Urls) Config() driver.Config { + return config +} + +func (d *Urls) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Urls) Init(ctx context.Context) error { + node, err := BuildTree(d.UrlStructure, d.HeadSize) + if err != nil { + return err + } + node.calSize() + d.root = node + return nil +} + +func (d *Urls) Drop(ctx context.Context) error { + return nil +} + +func (d *Urls) Get(ctx context.Context, path string) (model.Obj, error) { + node := GetNodeFromRootByPath(d.root, path) + return nodeToObj(node, path) +} + +func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + node := GetNodeFromRootByPath(d.root, dir.GetPath()) + log.Debugf("path: %s, node: %+v", dir.GetPath(), node) + if node == nil { + return nil, errs.ObjectNotFound + } + if node.isFile() { + return nil, errs.NotFolder + } + return utils.SliceConvert(node.Children, func(node *Node) (model.Obj, error) { + return nodeToObj(node, stdpath.Join(dir.GetPath(), node.Name)) + }) +} + +func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + node := GetNodeFromRootByPath(d.root, file.GetPath()) + log.Debugf("path: %s, node: %+v", file.GetPath(), node) + if node == nil { + return nil, errs.ObjectNotFound + } + if node.isFile() { + return &model.Link{ + URL: node.Url, + }, nil + } + return nil, errs.NotFile +} + +//func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Urls)(nil) diff --git a/drivers/url_tree/meta.go b/drivers/url_tree/meta.go new file mode 100644 index 00000000..b3ae33dc --- /dev/null +++ b/drivers/url_tree/meta.go @@ -0,0 +1,35 @@ +package url_tree + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + // driver.RootPath + // driver.RootID + // define other + UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://jsd.nn.ci/gh/alist-org/alist/README.md\nhttps://jsd.nn.ci/gh/alist-org/alist/README_cn.md\nfolder:\n CONTRIBUTING.md:1635:https://jsd.nn.ci/gh/alist-org/alist/CONTRIBUTING.md\n CODE_OF_CONDUCT.md:2093:https://jsd.nn.ci/gh/alist-org/alist/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n [FileName:][FileSize:][Modified:]Url"` + HeadSize bool `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."` +} + +var config = driver.Config{ + Name: "UrlTree", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: false, + NoCache: true, + NoUpload: true, + NeedMs: false, + DefaultRoot: "", + CheckStatus: true, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Urls{} + }) +} diff --git a/drivers/url_tree/types.go b/drivers/url_tree/types.go new file mode 100644 index 00000000..7e8ca3d9 --- /dev/null +++ b/drivers/url_tree/types.go @@ -0,0 +1,46 @@ +package url_tree + +// Node is a node in the folder tree +type Node struct { + Url string + Name string + Level int + Modified int64 + Size int64 + Children []*Node +} + +func (node *Node) getByPath(paths []string) *Node { + if len(paths) == 0 || node == nil { + return nil + } + if node.Name != paths[0] { + return nil + } + if len(paths) == 1 { + return node + } + for _, child := range node.Children { + tmp := child.getByPath(paths[1:]) + if tmp != nil { + return tmp + } + } + return nil +} + +func (node *Node) isFile() bool { + return node.Url != "" +} + +func (node *Node) calSize() int64 { + if node.isFile() { + return node.Size + } + var size int64 = 0 + for _, child := range node.Children { + size += child.calSize() + } + node.Size = size + return size +} diff --git a/drivers/url_tree/urls_test.go b/drivers/url_tree/urls_test.go new file mode 100644 index 00000000..a5314a3b --- /dev/null +++ b/drivers/url_tree/urls_test.go @@ -0,0 +1,47 @@ +package url_tree_test + +import ( + "testing" + + "github.com/alist-org/alist/v3/drivers/url_tree" +) + +func testTree() (*url_tree.Node, error) { + text := `folder1: + name1:url1 + url2 + folder2: + url3 + url4 + url5 +folder3: + url6 + url7 +url8` + return url_tree.BuildTree(text, false) +} + +func TestBuildTree(t *testing.T) { + node, err := testTree() + if err != nil { + t.Errorf("failed to build tree: %+v", err) + } else { + t.Logf("tree: %+v", node) + } +} + +func TestGetNode(t *testing.T) { + root, err := testTree() + if err != nil { + t.Errorf("failed to build tree: %+v", err) + return + } + node := url_tree.GetNodeFromRootByPath(root, "/") + if node != root { + t.Errorf("got wrong node: %+v", node) + } + url3 := url_tree.GetNodeFromRootByPath(root, "/folder1/folder2/url3") + if url3 != root.Children[0].Children[2].Children[0] { + t.Errorf("got wrong node: %+v", url3) + } +} diff --git a/drivers/url_tree/util.go b/drivers/url_tree/util.go new file mode 100644 index 00000000..0aa4dc83 --- /dev/null +++ b/drivers/url_tree/util.go @@ -0,0 +1,192 @@ +package url_tree + +import ( + "fmt" + stdpath "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + log "github.com/sirupsen/logrus" +) + +// build tree from text, text structure definition: +/** + * FolderName: + * [FileName:][FileSize:][Modified:]Url + */ +/** + * For example: + * folder1: + * name1:url1 + * url2 + * folder2: + * url3 + * url4 + * url5 + * folder3: + * url6 + * url7 + * url8 + */ +// if there are no name, use the last segment of url as name +func BuildTree(text string, headSize bool) (*Node, error) { + lines := strings.Split(text, "\n") + var root = &Node{Level: -1, Name: "root"} + stack := []*Node{root} + for _, line := range lines { + // calculate indent + indent := 0 + for i := 0; i < len(line); i++ { + if line[i] != ' ' { + break + } + indent++ + } + // if indent is not a multiple of 2, it is an error + if indent%2 != 0 { + return nil, fmt.Errorf("the line '%s' is not a multiple of 2", line) + } + // calculate level + level := indent / 2 + line = strings.TrimSpace(line[indent:]) + // if the line is empty, skip + if line == "" { + continue + } + // if level isn't greater than the level of the top of the stack + // it is not the child of the top of the stack + if level <= stack[len(stack)-1].Level { + // pop the top of the stack + stack = stack[:len(stack)-1] + } + // if the line is a folder + if isFolder(line) { + // create a new node + node := &Node{ + Level: level, + Name: strings.TrimSuffix(line, ":"), + } + // add the node to the top of the stack + stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node) + // push the node to the stack + stack = append(stack, node) + } else { + // if the line is a file + // create a new node + node, err := parseFileLine(line, headSize) + if err != nil { + return nil, err + } + node.Level = level + // add the node to the top of the stack + stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node) + } + } + return root, nil +} + +func isFolder(line string) bool { + return strings.HasSuffix(line, ":") +} + +// line definition: +// [FileName:][FileSize:][Modified:]Url +func parseFileLine(line string, headSize bool) (*Node, error) { + // if there is no url, it is an error + if !strings.Contains(line, "http://") && !strings.Contains(line, "https://") { + return nil, fmt.Errorf("invalid line: %s, because url is required for file", line) + } + index := strings.Index(line, "http://") + if index == -1 { + index = strings.Index(line, "https://") + } + url := line[index:] + info := line[:index] + node := &Node{ + Url: url, + } + haveSize := false + if index > 0 { + if !strings.HasSuffix(info, ":") { + return nil, fmt.Errorf("invalid line: %s, because file info must end with ':'", line) + } + info = info[:len(info)-1] + if info == "" { + return nil, fmt.Errorf("invalid line: %s, because file name can't be empty", line) + } + infoParts := strings.Split(info, ":") + node.Name = infoParts[0] + if len(infoParts) > 1 { + size, err := strconv.ParseInt(infoParts[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid line: %s, because file size must be an integer", line) + } + node.Size = size + haveSize = true + if len(infoParts) > 2 { + modified, err := strconv.ParseInt(infoParts[2], 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid line: %s, because file modified must be an unix timestamp", line) + } + node.Modified = modified + } + } + } else { + node.Name = stdpath.Base(url) + } + if !haveSize && headSize { + size, err := getSizeFromUrl(url) + if err != nil { + log.Errorf("get size from url error: %s", err) + } else { + node.Size = size + } + } + return node, nil +} + +func splitPath(path string) []string { + if path == "/" { + return []string{"root"} + } + parts := strings.Split(path, "/") + parts[0] = "root" + return parts +} + +func GetNodeFromRootByPath(root *Node, path string) *Node { + return root.getByPath(splitPath(path)) +} + +func nodeToObj(node *Node, path string) (model.Obj, error) { + if node == nil { + return nil, errs.ObjectNotFound + } + return &model.Object{ + Name: node.Name, + Size: node.Size, + Modified: time.Unix(node.Modified, 0), + IsFolder: !node.isFile(), + Path: path, + }, nil +} + +func getSizeFromUrl(url string) (int64, error) { + res, err := base.RestyClient.R().SetDoNotParseResponse(true).Head(url) + if err != nil { + return 0, err + } + defer res.RawResponse.Body.Close() + if res.StatusCode() >= 300 { + return 0, fmt.Errorf("get size from url %s failed, status code: %d", url, res.StatusCode()) + } + size, err := strconv.ParseInt(res.Header().Get("Content-Length"), 10, 64) + if err != nil { + return 0, err + } + return size, nil +} diff --git a/go.sum b/go.sum index 1ef75abc..67c12dc5 100644 --- a/go.sum +++ b/go.sum @@ -17,14 +17,10 @@ github.com/aws/aws-sdk-go v1.44.194 h1:1ZDK+QDcc5oRbZGgRZSz561eR8XVizXCeGpoZKo33 github.com/aws/aws-sdk-go v1.44.194/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/bleve/v2 v2.3.6 h1:NlntUHcV5CSWIhpugx4d/BRMGCiaoI8ZZXrXlahzNq4= -github.com/blevesearch/bleve/v2 v2.3.6/go.mod h1:JM2legf1cKVkdV8Ehu7msKIOKC0McSw0Q16Fmv9vsW4= github.com/blevesearch/bleve/v2 v2.3.7 h1:nIfIrhv28tvgBpbVF8Dq7/U1zW/YiwSqg/PBgE3x8bo= github.com/blevesearch/bleve/v2 v2.3.7/go.mod h1:2tToYD6mDeseIA13jcZiEEqYrVLg6xdk0v6+F7dWquU= github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw= github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= -github.com/blevesearch/geo v0.1.16 h1:unVaqUmlwprk56596OQRkGjtq1VZ8XFWSARj+h2cIBY= -github.com/blevesearch/geo v0.1.16/go.mod h1:a1OlySNE+oDQ5qY0vJGYNoLIsMpbKbx8dnmuRP8D7H0= github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA= github.com/blevesearch/geo v0.1.17/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= @@ -35,14 +31,10 @@ github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCD github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= github.com/blevesearch/scorch_segment_api/v2 v2.1.4 h1:LmGmo5twU3gV+natJbKmOktS9eMhokPGKWuR+jX84vk= github.com/blevesearch/scorch_segment_api/v2 v2.1.4/go.mod h1:PgVnbbg/t1UkgezPDu8EHLi1BHQ17xUwsFdU6NnOYS0= -github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= -github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= -github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU= -github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q= github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= github.com/blevesearch/vellum v1.0.9 h1:PL+NWVk3dDGPCV0hoDu9XLLJgqU4E5s/dOeEJByQ2uQ= @@ -55,8 +47,6 @@ github.com/blevesearch/zapx/v13 v13.3.7 h1:igIQg5eKmjw168I7av0Vtwedf7kHnQro/M+ub github.com/blevesearch/zapx/v13 v13.3.7/go.mod h1:yyrB4kJ0OT75UPZwT/zS+Ru0/jYKorCOOSY5dBzAy+s= github.com/blevesearch/zapx/v14 v14.3.7 h1:gfe+fbWslDWP/evHLtp/GOvmNM3sw1BbqD7LhycBX20= github.com/blevesearch/zapx/v14 v14.3.7/go.mod h1:9J/RbOkqZ1KSjmkOes03AkETX7hrXT0sFMpWH4ewC4w= -github.com/blevesearch/zapx/v15 v15.3.8 h1:q4uMngBHzL1IIhRc8AJUEkj6dGOE3u1l3phLu7hq8uk= -github.com/blevesearch/zapx/v15 v15.3.8/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY= github.com/blevesearch/zapx/v15 v15.3.9 h1:/s9zqKxFaZKQTTcMO2b/Tup0ch5MSztlvw+frVDfIBk= github.com/blevesearch/zapx/v15 v15.3.9/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= @@ -76,8 +66,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set/v2 v2.2.0 h1:2pMQd3Soi6qfw7E5MMKaEh5W5ES18bW3AbFFnGl6LgQ= -github.com/deckarep/golang-set/v2 v2.2.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g= github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= @@ -128,7 +116,6 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -162,7 +149,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=