perf: use io copy with buffer pool (#6389)

* feat: add io methods with buffer

* chore: move io.Copy calls to utils.CopyWithBuffer

pkg/gowebdav/client.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/xml"
 	"fmt"
+	"github.com/alist-org/alist/v3/pkg/utils"
 	"io"
 	"net/http"
 	"net/url"
@@ -419,7 +420,7 @@ func (c *Client) ReadStreamRange(path string, offset, length int64) (io.ReadClos
 	// stream in rs.Body
 	if rs.StatusCode == 200 {
 		// discard first 'offset' bytes.
-		if _, err := io.Copy(io.Discard, io.LimitReader(rs.Body, offset)); err != nil {
+		if _, err := utils.CopyWithBuffer(io.Discard, io.LimitReader(rs.Body, offset)); err != nil {
 			return nil, newPathErrorErr("ReadStreamRange", path, err)
 		}
 

pkg/utils/file.go

@@ -32,7 +32,7 @@ func CopyFile(src, dst string) error {
 	}
 	defer dstfd.Close()
 
-	if _, err = io.Copy(dstfd, srcfd); err != nil {
+	if _, err = CopyWithBuffer(dstfd, srcfd); err != nil {
 		return err
 	}
 	if srcinfo, err = os.Stat(src); err != nil {
@@ -121,7 +121,7 @@ func CreateTempFile(r io.Reader, size int64) (*os.File, error) {
 	if err != nil {
 		return nil, err
 	}
-	readBytes, err := io.Copy(f, r)
+	readBytes, err := CopyWithBuffer(f, r)
 	if err != nil {
 		_ = os.Remove(f.Name())
 		return nil, errs.NewErr(err, "CreateTempFile failed")

pkg/utils/hash.go

@@ -96,7 +96,7 @@ func HashData(hashType *HashType, data []byte, params ...any) string {
 // HashReader get hash of one hashType from a reader
 func HashReader(hashType *HashType, reader io.Reader, params ...any) (string, error) {
 	h := hashType.NewFunc(params...)
-	_, err := io.Copy(h, reader)
+	_, err := CopyWithBuffer(h, reader)
 	if err != nil {
 		return "", errs.NewErr(err, "HashReader error")
 	}

pkg/utils/hash_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"io"
 	"testing"
 )
 
@@ -36,7 +35,7 @@ var hashTestSet = []hashTest{
 func TestMultiHasher(t *testing.T) {
 	for _, test := range hashTestSet {
 		mh := NewMultiHasher([]*HashType{MD5, SHA1, SHA256})
-		n, err := io.Copy(mh, bytes.NewBuffer(test.input))
+		n, err := CopyWithBuffer(mh, bytes.NewBuffer(test.input))
 		require.NoError(t, err)
 		assert.Len(t, test.input, int(n))
 		hashInfo := mh.GetHashInfo()

pkg/utils/io.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"sync"
 	"time"
 
 	"golang.org/x/exp/constraints"
@@ -29,7 +30,7 @@ func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, p
 	// possible in the call process.
 	var finish int64 = 0
 	s := size / 100
-	_, err := io.Copy(out, readerFunc(func(p []byte) (int, error) {
+	_, err := CopyWithBuffer(out, readerFunc(func(p []byte) (int, error) {
 		// golang non-blocking channel: https://gobyexample.com/non-blocking-channel-operations
 		select {
 		// if context has been canceled
@@ -204,3 +205,31 @@ func Max[T constraints.Ordered](a, b T) T {
 	}
 	return a
 }
+
+var IoBuffPool = &sync.Pool{
+	New: func() interface{} {
+		return make([]byte, 32*1024*2) // Two times of size in io package
+	},
+}
+
+func CopyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) {
+	buff := IoBuffPool.Get().([]byte)
+	defer IoBuffPool.Put(buff)
+	written, err = io.CopyBuffer(dst, src, buff)
+	if err != nil {
+		return
+	}
+	return written, nil
+}
+
+func CopyWithBufferN(dst io.Writer, src io.Reader, n int64) (written int64, err error) {
+	written, err = CopyWithBuffer(dst, io.LimitReader(src, n))
+	if written == n {
+		return n, nil
+	}
+	if written < n && err == nil {
+		// src stopped early; must have been EOF.
+		err = io.EOF
+	}
+	return
+}