eos

package module
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 25, 2024 License: Apache-2.0 Imports: 35 Imported by: 2

README

EOSS: Wrapper For Aliyun OSS And Amazon S3

awos for node: https://github.com/shimohq/awos-js

Features

  • enable shards bucket
  • add retry strategy
  • avoid 404 status code:
    • Get(objectName string) (string, error) will return "", nil when object not exist
    • Head(key string, meta []string) (map[string]string, error) will return nil, nil when object not exist

Installing

Use go get to retrieve the SDK to add it to your GOPATH workspace, or project's Go module dependencies.

go get github.com/ego-component/eos

How to use

config
[storage]
storageType = "oss" # oss|s3
accessKeyID = "xxx"
accessKeySecret = "xxx"
endpoint = "oss-cn-beijing.aliyuncs.com"
bucket = "aaa" # 定义默认storage实例
shards = []
  # 定义其他storage实例
  [storage.buckets.template] 
  bucket = "template-bucket"
  shards = []
  [storage.buckets.fileContent]
  bucket = "contents-bucket"
  shards = [
   "abcdefghijklmnopqr",
   "stuvwxyz0123456789"
  ]

import "github.com/ego-component/eos"

// 构建 os component
cmp := eoss.Load("storage").Build()

Available operations：

Get(ctx context.Context, key string, options ...GetOptions) (string, error)
GetBytes(ctx context.Context, key string, options ...GetOptions) ([]byte, error)
GetAsReader(ctx context.Context, key string, options ...GetOptions) (io.ReadCloser, error)
GetWithMeta(ctx context.Context, key string, attributes []string, options ...GetOptions) (io.ReadCloser, map[string]string, error)
Put(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error
Del(ctx context.Context, key string) error
DelMulti(ctx context.Context, keys []string) error
Head(ctx context.Context, key string, meta []string) (map[string]string, error)
ListObject(ctx context.Context, key string, prefix string, marker string, maxKeys int, delimiter string) ([]string, error)
SignURL(ctx context.Context, key string, expired int64) (string, error)
GetAndDecompress(ctx context.Context, key string) (string, error)
GetAndDecompressAsReader(ctx context.Context, key string) (io.ReadCloser, error)
CompressAndPut(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error
Range(ctx context.Context, key string, offset int64, length int64) (io.ReadCloser, error)
Exists(ctx context.Context, key string)(bool, error)

Documentation

Index

Constants

View Source 
const (
	StorageTypeOSS  = "oss"
	StorageTypeS3   = "s3"
	StorageTypeFile = "file"

	MetaCompressor = "compressor"
)
View Source 
const PackageName = "component.eoss"

Variables

View Source 
var DefaultGzipCompressor = &GzipCompressor{}

Functions

func DefaultConfig 

func DefaultConfig() *config

DefaultConfig 返回默认配置

func DefaultCopyOptions 

func DefaultCopyOptions() *copyOptions

func DefaultGetOptions 

func DefaultGetOptions() *getOptions

func DefaultPutOptions 

func DefaultPutOptions() *putOptions

func DefaultSignOptions 

func DefaultSignOptions() *signOptions

func GetReaderLength 

func GetReaderLength(reader io.ReadSeeker) (int64, error)

func Register 

func Register(comp Compressor)

func WrapReader added in v0.1.1 

func WrapReader(reader io.ReadSeeker) (io.ReadSeeker, int64, error)

Types

type BucketConfig 

type BucketConfig struct {
	// Debug
	Debug bool
	// Required, value is one of oss/s3, case insensetive
	StorageType string
	// Required
	AccessKeyID string
	// Required
	AccessKeySecret string
	// Required
	Endpoint string
	// Required Bucket name
	Bucket string
	// Prefix sets a default global prefix, if not empty, this prefix will automatically be added to all keys by default
	Prefix string
	// Optional, choose which bucket to use based on the last character of the key,
	// if bucket is 'content', shards is ['abc', 'edf'],
	// then the last character of the key with a/b/c will automatically use the content-abc bucket, and vice versa
	Shards []string
	// Only for s3-like
	Region string
	// Only for s3-like, whether to force path style URLs for S3 objects.
	S3ForcePathStyle bool
	// Only for s3-like
	SSL bool
	// Only for s3-like, set http client timeout.
	// oss has default timeout, but s3 default timeout is 0 means no timeout.
	S3HttpTimeoutSecs int64
	// EnableTraceInterceptor enable otel trace (only for s3)
	EnableTraceInterceptor bool
	// EnableMetricInterceptor enable prom metrics
	EnableMetricInterceptor bool
	// EnableClientTrace
	EnableClientTrace bool
	// EnableCompressor
	EnableCompressor bool
	// CompressType gzip
	CompressType string
	// CompressLimit 大于该值之后才压缩 单位字节
	CompressLimit int64
	// MaxIdleConns 设置最大空闲连接数
	MaxIdleConns int
	// MaxIdleConnsPerHost 设置长连接个数
	MaxIdleConnsPerHost int
	// EnableKeepAlives 是否开启长连接，默认打开
	EnableKeepAlives bool
	// IdleConnTimeout 设置空闲连接时间，默认90 * time.Second
	IdleConnTimeout time.Duration
}

type BuildOption 

type BuildOption func(c *Container)

func WithAccessKeyID 

func WithAccessKeyID(ak string) BuildOption

func WithAccessKeySecret 

func WithAccessKeySecret(sk string) BuildOption

func WithBucket 

func WithBucket(bucket string) BuildOption

func WithDebug 

func WithDebug(debug bool) BuildOption

func WithEndpoint 

func WithEndpoint(endpoint string) BuildOption

func WithIdleConnTimeout added in v0.1.1 

func WithIdleConnTimeout(idleConnTimeout time.Duration) BuildOption

func WithKeepAlives added in v0.1.1 

func WithKeepAlives(enableKeepAlives bool) BuildOption

func WithMaxIdleConns added in v0.1.1 

func WithMaxIdleConns(maxIdleConns int) BuildOption

func WithMaxIdleConnsPerHost added in v0.1.1 

func WithMaxIdleConnsPerHost(maxIdleConnsPerHost int) BuildOption

func WithRegion 

func WithRegion(region string) BuildOption

func WithS3ForcePathStyle 

func WithS3ForcePathStyle(s3ForcePathStyle bool) BuildOption

func WithS3HttpTimeoutSecs 

func WithS3HttpTimeoutSecs(s3HttpTimeoutSecs int64) BuildOption

func WithSSL 

func WithSSL(ssl bool) BuildOption

func WithShards 

func WithShards(shards []string) BuildOption

func WithStorageType 

func WithStorageType(storageType string) BuildOption

type Client 

type Client interface {
	GetBucketName(ctx context.Context, key string) (string, error)
	Get(ctx context.Context, key string, options ...GetOptions) (string, error)
	GetBytes(ctx context.Context, key string, options ...GetOptions) ([]byte, error)
	GetAsReader(ctx context.Context, key string, options ...GetOptions) (io.ReadCloser, error)
	GetWithMeta(ctx context.Context, key string, attributes []string, options ...GetOptions) (io.ReadCloser, map[string]string, error)
	GetAndDecompress(ctx context.Context, key string) (string, error)
	GetAndDecompressAsReader(ctx context.Context, key string) (io.ReadCloser, error)
	Put(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error
	PutAndCompress(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error
	Del(ctx context.Context, key string) error
	DelMulti(ctx context.Context, keys []string) error
	Head(ctx context.Context, key string, meta []string) (map[string]string, error)
	ListObject(ctx context.Context, key string, prefix string, marker string, maxKeys int, delimiter string) ([]string, error)
	SignURL(ctx context.Context, key string, expired int64, options ...SignOptions) (string, error)
	Range(ctx context.Context, key string, offset int64, length int64) (io.ReadCloser, error)
	Exists(ctx context.Context, key string) (bool, error)
	Copy(ctx context.Context, srcKey, dstKey string, options ...CopyOption) error
}

Client object storage client interface

type CombinedReadCloser 

type CombinedReadCloser struct {
	ReadCloser io.ReadCloser
	Reader     io.Reader
}

CombinedReadCloser combined a ReadCloser and a Readers to a new ReaderCloser which will read from reader and close origin closer

func (CombinedReadCloser) Close 

func (combined CombinedReadCloser) Close() error

Close origin ReaderCloser

func (CombinedReadCloser) Read 

func (combined CombinedReadCloser) Read(b []byte) (int, error)

type Component 

type Component struct {
	// contains filtered or unexported fields
}

func (*Component) Client 

func (c *Component) Client(bucket string) Client

Client return specific storage client instance

func (*Component) Copy 

func (c *Component) Copy(ctx context.Context, srcKey, dstKey string, options ...CopyOption) error

func (*Component) DefaultClient 

func (c *Component) DefaultClient() Client

DefaultClient return default storage client

func (*Component) Del 

func (c *Component) Del(ctx context.Context, key string) error

func (*Component) DelMulti 

func (c *Component) DelMulti(ctx context.Context, keys []string) error

func (*Component) Exists 

func (c *Component) Exists(ctx context.Context, key string) (bool, error)

func (*Component) Get 

func (c *Component) Get(ctx context.Context, key string, options ...GetOptions) (string, error)

func (*Component) GetAndDecompress 

func (c *Component) GetAndDecompress(ctx context.Context, key string) (string, error)

func (*Component) GetAndDecompressAsReader 

func (c *Component) GetAndDecompressAsReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*Component) GetAsReader 

func (c *Component) GetAsReader(ctx context.Context, key string, options ...GetOptions) (io.ReadCloser, error)

GetAsReader don't forget to call the close() method of the io.ReadCloser

func (*Component) GetBucketName 

func (c *Component) GetBucketName(ctx context.Context, key string) (string, error)

func (*Component) GetBytes 

func (c *Component) GetBytes(ctx context.Context, key string, options ...GetOptions) ([]byte, error)

func (*Component) GetWithMeta 

func (c *Component) GetWithMeta(ctx context.Context, key string, attributes []string, options ...GetOptions) (io.ReadCloser, map[string]string, error)

GetWithMeta don't forget to call the close() method of the io.ReadCloser

func (*Component) Head 

func (c *Component) Head(ctx context.Context, key string, attributes []string) (map[string]string, error)

func (*Component) ListObject 

func (c *Component) ListObject(ctx context.Context, key string, prefix string, marker string, maxKeys int, delimiter string) ([]string, error)

func (*Component) Put 

func (c *Component) Put(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error

func (*Component) PutAndCompress 

func (c *Component) PutAndCompress(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error

func (*Component) Range 

func (c *Component) Range(ctx context.Context, key string, offset int64, length int64) (io.ReadCloser, error)

func (*Component) SignURL 

func (c *Component) SignURL(ctx context.Context, key string, expired int64, options ...SignOptions) (string, error)

type Compressor 

type Compressor interface {
	// Compress(reader io.ReadSeeker) (gzipReader io.ReadSeeker, err error)
	Compress(reader io.ReadSeeker) (gzipReader io.ReadSeeker, len int64, err error)
	ContentEncoding() string
}

type Container 

type Container struct {
	// contains filtered or unexported fields
}

func DefaultContainer 

func DefaultContainer() *Container

func Load 

func Load(key string) *Container

func (*Container) Build 

func (c *Container) Build(options ...BuildOption) *Component

type CopyOption 

type CopyOption func(options *copyOptions)

func CopyWithAttributes 

func CopyWithAttributes(meta []string) CopyOption

CopyWithAttributes specify metadata keys to copy

func CopyWithNewAttributes 

func CopyWithNewAttributes(meta map[string]string) CopyOption

CopyWithNewAttributes append new attributes(meta) to new object

NOTE: if this option was specified, the metadata(s) of source object would be dropped expect specifying keys to copy using CopyWithAttributes() option.

func CopyWithRawSrcKey 

func CopyWithRawSrcKey() CopyOption

type GetOptions 

type GetOptions func(options *getOptions)

func EnableCRCValidation 

func EnableCRCValidation() GetOptions

func GetWithContentEncoding 

func GetWithContentEncoding(contentEncoding string) GetOptions

func GetWithContentType 

func GetWithContentType(contentType string) GetOptions

type GzipCompressor 

type GzipCompressor struct {
}

func (*GzipCompressor) Compress 

func (g *GzipCompressor) Compress(reader io.ReadSeeker) (gzipReader io.ReadSeeker, len int64, err error)

func (*GzipCompressor) ContentEncoding 

func (g *GzipCompressor) ContentEncoding() string

type HeadGetObjectOutputWrapper 

type HeadGetObjectOutputWrapper struct {
	// contains filtered or unexported fields
}

type LocalFile added in v1.0.0 

type LocalFile struct {
	// contains filtered or unexported fields
}

LocalFile is the implementation based on local files. For desktop APP or test.

func NewLocalFile added in v1.0.0 

func NewLocalFile(path string) (*LocalFile, error)

func (*LocalFile) Copy added in v1.0.0 

func (l *LocalFile) Copy(ctx context.Context, srcKey, dstKey string, options ...CopyOption) error

func (*LocalFile) Del added in v1.0.0 

func (l *LocalFile) Del(ctx context.Context, key string) error

func (*LocalFile) DelMulti added in v1.0.0 

func (l *LocalFile) DelMulti(ctx context.Context, keys []string) error

func (*LocalFile) Exists added in v1.0.0 

func (l *LocalFile) Exists(ctx context.Context, key string) (bool, error)

func (*LocalFile) Get added in v1.0.0 

func (l *LocalFile) Get(ctx context.Context, key string, options ...GetOptions) (string, error)

func (*LocalFile) GetAndDecompress added in v1.0.0 

func (l *LocalFile) GetAndDecompress(ctx context.Context, key string) (string, error)

func (*LocalFile) GetAndDecompressAsReader added in v1.0.0 

func (l *LocalFile) GetAndDecompressAsReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*LocalFile) GetAsReader added in v1.0.0 

func (l *LocalFile) GetAsReader(ctx context.Context, key string, options ...GetOptions) (io.ReadCloser, error)

GetAsReader returns reader which you need to close it.

func (*LocalFile) GetBucketName added in v1.0.0 

func (l *LocalFile) GetBucketName(ctx context.Context, key string) (string, error)

func (*LocalFile) GetBytes added in v1.0.0 

func (l *LocalFile) GetBytes(ctx context.Context, key string, options ...GetOptions) ([]byte, error)

func (*LocalFile) GetWithMeta added in v1.0.0 

func (l *LocalFile) GetWithMeta(ctx context.Context, key string, attributes []string, options ...GetOptions) (io.ReadCloser, map[string]string, error)

func (*LocalFile) Head added in v1.0.0 

func (l *LocalFile) Head(ctx context.Context, key string, attributes []string) (map[string]string, error)

func (*LocalFile) ListObject added in v1.0.0 

func (l *LocalFile) ListObject(ctx context.Context, key string, prefix string, marker string, maxKeys int, delimiter string) ([]string, error)

func (*LocalFile) Put added in v1.0.0 

func (l *LocalFile) Put(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error

Put override the file It will create two files, one for content, one for meta.

func (*LocalFile) PutAndCompress added in v1.0.0 

func (l *LocalFile) PutAndCompress(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error

func (*LocalFile) Range added in v1.0.0 

func (l *LocalFile) Range(ctx context.Context, key string, offset int64, length int64) (io.ReadCloser, error)

func (*LocalFile) SignURL added in v1.0.0 

func (l *LocalFile) SignURL(ctx context.Context, key string, expired int64, options ...SignOptions) (string, error)

type OSS 

type OSS struct {
	Bucket *oss.Bucket
	Shards map[string]*oss.Bucket
	// contains filtered or unexported fields
}

func (*OSS) Copy 

func (ossClient *OSS) Copy(ctx context.Context, srcKey, dstKey string, options ...CopyOption) error

func (*OSS) Del 

func (ossClient *OSS) Del(ctx context.Context, key string) error

func (*OSS) DelMulti 

func (ossClient *OSS) DelMulti(ctx context.Context, keys []string) error

func (*OSS) Exists 

func (ossClient *OSS) Exists(ctx context.Context, key string) (bool, error)

func (*OSS) Get 

func (ossClient *OSS) Get(ctx context.Context, key string, options ...GetOptions) (string, error)

func (*OSS) GetAndDecompress 

func (ossClient *OSS) GetAndDecompress(ctx context.Context, key string) (string, error)

func (*OSS) GetAndDecompressAsReader 

func (ossClient *OSS) GetAndDecompressAsReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*OSS) GetAsReader 

func (ossClient *OSS) GetAsReader(ctx context.Context, key string, options ...GetOptions) (io.ReadCloser, error)

GetAsReader don't forget to call the close() method of the io.ReadCloser

func (*OSS) GetBucketName 

func (ossClient *OSS) GetBucketName(ctx context.Context, key string) (string, error)

func (*OSS) GetBytes 

func (ossClient *OSS) GetBytes(ctx context.Context, key string, options ...GetOptions) ([]byte, error)

func (*OSS) GetWithMeta 

func (ossClient *OSS) GetWithMeta(ctx context.Context, key string, attributes []string, options ...GetOptions) (io.ReadCloser, map[string]string, error)

GetWithMeta don't forget to call the close() method of the io.ReadCloser

func (*OSS) Head 

func (ossClient *OSS) Head(ctx context.Context, key string, attributes []string) (map[string]string, error)

func (*OSS) ListObject 

func (ossClient *OSS) ListObject(ctx context.Context, key string, prefix string, marker string, maxKeys int, delimiter string) ([]string, error)

func (*OSS) Put 

func (ossClient *OSS) Put(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error

func (*OSS) PutAndCompress 

func (ossClient *OSS) PutAndCompress(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error

func (*OSS) Range 

func (ossClient *OSS) Range(ctx context.Context, key string, offset int64, length int64) (io.ReadCloser, error)

func (*OSS) SignURL 

func (ossClient *OSS) SignURL(ctx context.Context, key string, expired int64, options ...SignOptions) (string, error)

type PutOptions 

type PutOptions func(options *putOptions)

func PutWithCacheControl 

func PutWithCacheControl(cacheControl string) PutOptions

func PutWithContentDisposition 

func PutWithContentDisposition(contentDisposition string) PutOptions

func PutWithContentEncoding 

func PutWithContentEncoding(contentEncoding string) PutOptions

func PutWithContentType 

func PutWithContentType(contentType string) PutOptions

func PutWithExpireTime 

func PutWithExpireTime(expires time.Time) PutOptions

type S3 

type S3 struct {
	ShardsBucket map[string]string
	BucketName   string
	// contains filtered or unexported fields
}

func (*S3) Copy 

func (a *S3) Copy(ctx context.Context, srcKey, dstKey string, options ...CopyOption) error

func (*S3) Del 

func (a *S3) Del(ctx context.Context, key string) error

func (*S3) DelMulti 

func (a *S3) DelMulti(ctx context.Context, keys []string) error

func (*S3) Exists 

func (a *S3) Exists(ctx context.Context, key string) (bool, error)

func (*S3) Get 

func (a *S3) Get(ctx context.Context, key string, options ...GetOptions) (string, error)

func (*S3) GetAndDecompress 

func (a *S3) GetAndDecompress(ctx context.Context, key string) (string, error)

func (*S3) GetAndDecompressAsReader 

func (a *S3) GetAndDecompressAsReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*S3) GetAsReader 

func (a *S3) GetAsReader(ctx context.Context, key string, options ...GetOptions) (io.ReadCloser, error)

GetAsReader don't forget to call the close() method of the io.ReadCloser

func (*S3) GetBucketName 

func (a *S3) GetBucketName(ctx context.Context, key string) (string, error)

func (*S3) GetBytes 

func (a *S3) GetBytes(ctx context.Context, key string, options ...GetOptions) ([]byte, error)

func (*S3) GetWithMeta 

func (a *S3) GetWithMeta(ctx context.Context, key string, attributes []string, options ...GetOptions) (io.ReadCloser, map[string]string, error)

GetWithMeta don't forget to call the close() method of the io.ReadCloser

func (*S3) Head 

func (a *S3) Head(ctx context.Context, key string, attributes []string) (map[string]string, error)

func (*S3) ListObject 

func (a *S3) ListObject(ctx context.Context, key string, prefix string, marker string, maxKeys int, delimiter string) ([]string, error)

func (*S3) Put 

func (a *S3) Put(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error

func (*S3) PutAndCompress 

func (a *S3) PutAndCompress(ctx context.Context, key string, reader io.ReadSeeker, meta map[string]string, options ...PutOptions) error

func (*S3) Range 

func (a *S3) Range(ctx context.Context, key string, offset int64, length int64) (io.ReadCloser, error)

func (*S3) SignURL 

func (a *S3) SignURL(ctx context.Context, key string, expired int64, options ...SignOptions) (string, error)

type SignOptions 

type SignOptions func(options *signOptions)

func SignWithProcess 

func SignWithProcess(process string) SignOptions

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.