Add object storage module to make related operations just like operating on system files

modules/storage/minio.go

@@ -0,0 +1,228 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package storage
+
+import (
+	"context"
+	"io"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	core_module "github.com/oj-lab/platform/modules/core"
+)
+
+var (
+	_ ObjectStorage = &MinioStorage{}
+
+	quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
+)
+
+type minioObject struct {
+	*minio.Object
+}
+
+func (m *minioObject) Stat() (os.FileInfo, error) {
+	oi, err := m.Object.Stat()
+	if err != nil {
+		return nil, convertMinioErr(err)
+	}
+
+	return &minioFileInfo{oi}, nil
+}
+
+// MinioStorage returns a minio bucket storage
+type MinioStorage struct {
+	ctx      context.Context
+	client   *minio.Client
+	bucket   string
+	basePath string
+}
+
+func convertMinioErr(err error) error {
+	if err == nil {
+		return nil
+	}
+	errResp, ok := err.(minio.ErrorResponse)
+	if !ok {
+		return err
+	}
+
+	// Convert two responses to standard analogues
+	switch errResp.Code {
+	case "NoSuchKey":
+		return os.ErrNotExist
+	case "AccessDenied":
+		return os.ErrPermission
+	}
+
+	return err
+}
+
+// NewMinioStorage returns a minio storage
+func NewMinioStorage(ctx context.Context) (ObjectStorage, error) {
+	endpoint := core_module.Config.GetString(objectStorageEndpointConfigKey)
+	accessKeyID := core_module.Config.GetString(objectStorageAccessKeyConfigKey)
+	secretAccessKey := core_module.Config.GetString(objectStorageSecretKeyConfigKey)
+
+	client, err := minio.New(endpoint, &minio.Options{
+		Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+	})
+	if err != nil {
+		return nil, convertMinioErr(err)
+	}
+
+	bucket := core_module.Config.GetString(objectStorageBucketConfigKey)
+	basePath := core_module.Config.GetString(objectStorageBasePathConfigKey)
+
+	// Check to see if we already own this bucket
+	exists, err := client.BucketExists(ctx, bucket)
+	if err != nil {
+		return nil, convertMinioErr(err)
+	}
+	if !exists {
+		if err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
+			return nil, convertMinioErr(err)
+		}
+	}
+
+	return &MinioStorage{
+		ctx:      ctx,
+		client:   client,
+		bucket:   bucket,
+		basePath: basePath,
+	}, nil
+}
+
+func (m *MinioStorage) buildMinioPath(p string) string {
+	p = strings.TrimPrefix(path.Join(m.basePath, p), "/") // object store doesn't use slash for root path
+	if p == "." {
+		p = "" // object store doesn't use dot as relative path
+	}
+	return p
+}
+
+func (m *MinioStorage) buildMinioDirPrefix(p string) string {
+	// ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo"
+	p = m.buildMinioPath(p) + "/"
+	if p == "/" {
+		p = "" // object store doesn't use slash for root path
+	}
+	return p
+}
+
+// Open opens a file
+func (m *MinioStorage) Open(path string) (Object, error) {
+	opts := minio.GetObjectOptions{}
+	object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
+	if err != nil {
+		return nil, convertMinioErr(err)
+	}
+	return &minioObject{object}, nil
+}
+
+// Save saves a file to minio
+func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) {
+	uploadInfo, err := m.client.PutObject(
+		m.ctx,
+		m.bucket,
+		m.buildMinioPath(path),
+		r,
+		size,
+		minio.PutObjectOptions{
+			ContentType: "application/octet-stream",
+		},
+	)
+	if err != nil {
+		return 0, convertMinioErr(err)
+	}
+	return uploadInfo.Size, nil
+}
+
+type minioFileInfo struct {
+	minio.ObjectInfo
+}
+
+func (m minioFileInfo) Name() string {
+	return path.Base(m.ObjectInfo.Key)
+}
+
+func (m minioFileInfo) Size() int64 {
+	return m.ObjectInfo.Size
+}
+
+func (m minioFileInfo) ModTime() time.Time {
+	return m.LastModified
+}
+
+func (m minioFileInfo) IsDir() bool {
+	return strings.HasSuffix(m.ObjectInfo.Key, "/")
+}
+
+func (m minioFileInfo) Mode() os.FileMode {
+	return os.ModePerm
+}
+
+func (m minioFileInfo) Sys() any {
+	return nil
+}
+
+// Stat returns the stat information of the object
+func (m *MinioStorage) Stat(path string) (os.FileInfo, error) {
+	info, err := m.client.StatObject(
+		m.ctx,
+		m.bucket,
+		m.buildMinioPath(path),
+		minio.StatObjectOptions{},
+	)
+	if err != nil {
+		return nil, convertMinioErr(err)
+	}
+	return &minioFileInfo{info}, nil
+}
+
+// Delete delete a file
+func (m *MinioStorage) Delete(path string) error {
+	err := m.client.RemoveObject(m.ctx, m.bucket, m.buildMinioPath(path), minio.RemoveObjectOptions{})
+
+	return convertMinioErr(err)
+}
+
+// URL gets the redirect URL to a file. The presigned link is valid for 5 minutes.
+func (m *MinioStorage) URL(path, name string, serveDirectReqParams url.Values) (*url.URL, error) {
+	// copy serveDirectReqParams
+	reqParams, err := url.ParseQuery(serveDirectReqParams.Encode())
+	if err != nil {
+		return nil, err
+	}
+	// TODO it may be good to embed images with 'inline' like ServeData does, but we don't want to have to read the file, do we?
+	reqParams.Set("response-content-disposition", "attachment; filename=\""+quoteEscaper.Replace(name)+"\"")
+	u, err := m.client.PresignedGetObject(m.ctx, m.bucket, m.buildMinioPath(path), 5*time.Minute, reqParams)
+	return u, convertMinioErr(err)
+}
+
+// IterateObjects iterates across the objects in the miniostorage
+func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
+	opts := minio.GetObjectOptions{}
+	for mObjInfo := range m.client.ListObjects(m.ctx, m.bucket, minio.ListObjectsOptions{
+		Prefix:    m.buildMinioDirPrefix(dirName),
+		Recursive: true,
+	}) {
+		object, err := m.client.GetObject(m.ctx, m.bucket, mObjInfo.Key, opts)
+		if err != nil {
+			return convertMinioErr(err)
+		}
+		if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
+			defer object.Close()
+			return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object})
+		}(object, fn); err != nil {
+			return convertMinioErr(err)
+		}
+	}
+	return nil
+}

modules/storage/storage.go

@@ -0,0 +1,73 @@
+package storage
+
+import (
+	"io"
+	"net/url"
+	"os"
+)
+
+const (
+	objectStorageEndpointConfigKey  = "object_storage.endpoint"
+	objectStorageAccessKeyConfigKey = "object_storage.access_key"
+	objectStorageSecretKeyConfigKey = "object_storage.secret_key"
+	objectStorageBucketConfigKey    = "object_storage.bucket"
+	objectStorageBasePathConfigKey  = "object_storage.base_path"
+)
+
+// Object represents the object on the storage
+type Object interface {
+	io.ReadCloser
+	io.Seeker
+	Stat() (os.FileInfo, error)
+}
+
+// ObjectStorage represents an object storage to handle a bucket and files
+type ObjectStorage interface {
+	Open(path string) (Object, error)
+	// Save store a object, if size is unknown set -1
+	Save(path string, r io.Reader, size int64) (int64, error)
+	Stat(path string) (os.FileInfo, error)
+	Delete(path string) error
+	URL(path, name string, reqParams url.Values) (*url.URL, error)
+	IterateObjects(path string, iterator func(path string, obj Object) error) error
+}
+
+// Copy copies a file from source ObjectStorage to dest ObjectStorage
+func Copy(dstStorage ObjectStorage, dstPath string, srcStorage ObjectStorage, srcPath string) (int64, error) {
+	f, err := srcStorage.Open(srcPath)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+
+	size := int64(-1)
+	fsinfo, err := f.Stat()
+	if err == nil {
+		size = fsinfo.Size()
+	}
+
+	return dstStorage.Save(dstPath, f, size)
+}
+
+// Clean delete all the objects in this storage
+func Clean(storage ObjectStorage) error {
+	return storage.IterateObjects("", func(path string, obj Object) error {
+		_ = obj.Close()
+		return storage.Delete(path)
+	})
+}
+
+// SaveFrom saves data to the ObjectStorage with path p from the callback
+func SaveFrom(objStorage ObjectStorage, path string, callback func(w io.Writer) error) error {
+	pr, pw := io.Pipe()
+	defer pr.Close()
+	go func() {
+		defer pw.Close()
+		if err := callback(pw); err != nil {
+			_ = pw.CloseWithError(err)
+		}
+	}()
+
+	_, err := objStorage.Save(path, pr, -1)
+	return err
+}