storage

README

Package storage v1.2.1

Documentation

Index

• Constants
• func WithPartSize(size int64) s3Opt
• func WithRetryPolicy(policy RetryPolicy) s3Opt
• func WithUploadConcurrency(concurrency int) s3Opt
• type Backend
• type BackendMethod
• type ObjectNotFound
  • func (o ObjectNotFound) Error() string
• type RetryPolicy
• type S3
  • func NewS3(cfg S3Config, opts ...s3Opt) *S3
  • func (s *S3) Delete(ctx context.Context, path string) error
  • func (s *S3) Get(ctx context.Context, path string) (io.ReadCloser, error)
  • func (s *S3) Info(ctx context.Context, path string) (types.Info, error)
  • func (s *S3) Size(ctx context.Context, path string) (int64, error)
  • func (s *S3) Upload(ctx context.Context, file io.Reader, path string) error
• type S3Client
• type S3Config
• type Swift
  • func NewSwift(cfg SwiftConfig) (*Swift, error)
  • func (s *Swift) Delete(ctx context.Context, path string) error
  • func (s *Swift) Get(ctx context.Context, path string) (io.ReadCloser, error)
  • func (s *Swift) Info(ctx context.Context, path string) (types.Info, error)
  • func (s *Swift) Size(ctx context.Context, path string) (int64, error)
  • func (s *Swift) Upload(ctx context.Context, reader io.Reader, path string) error
• type SwiftConfig

Constants

const (
	NotFoundErrCode = "NotFound"
	// DefaultPartSize 16 MB part size define the size in bytes of the parts
	// uploaded in a multipart upload
	DefaultPartSize = int64(16777216)
	// DefaultUploadConcurrency defines that multipart upload will be done in
	// parallel in 2 routines
	DefaultUploadConcurrency = int(2)
)

Functions

func WithPartSize

func WithPartSize(size int64) s3Opt

func WithRetryPolicy

func WithRetryPolicy(policy RetryPolicy) s3Opt

WithRetryPolicy is an option to constructor NewS3 to add a Retry Policy impacting GET operations

func WithUploadConcurrency

func WithUploadConcurrency(concurrency int) s3Opt

Types

type Backend

type Backend interface {
	Get(ctx context.Context, path string) (io.ReadCloser, error)
	Upload(ctx context.Context, file io.Reader, path string) error
	Size(ctx context.Context, path string) (int64, error)
	Delete(ctx context.Context, path string) error
	Info(ctx context.Context, path string) (types.Info, error)
}

Backend represents something which is able to store files on an object storage service

type BackendMethod

type BackendMethod string

BackendMethod represents the name of a Method included in the Backend interface

const (
	GetMethod    BackendMethod = "Get"
	UploadMethod BackendMethod = "Upload"
	SizeMethod   BackendMethod = "Size"
	InfoMethod   BackendMethod = "Info"
	DeleteMethod BackendMethod = "Delete"
)

type ObjectNotFound

type ObjectNotFound struct{}

ObjectNotFound is a custom error in case the object is not found

func (ObjectNotFound) Error

func (o ObjectNotFound) Error() string

type RetryPolicy

type RetryPolicy struct {
	WaitDuration   time.Duration
	Attempts       int
	MethodHandlers map[BackendMethod][]string
}

type S3

type S3 struct {
	// contains filtered or unexported fields
}

func NewS3

func NewS3(cfg S3Config, opts ...s3Opt) *S3

func (*S3) Delete

func (s *S3) Delete(ctx context.Context, path string) error

func (*S3) Get

func (s *S3) Get(ctx context.Context, path string) (io.ReadCloser, error)

func (*S3) Info

func (s *S3) Info(ctx context.Context, path string) (types.Info, error)

Info returns several information contained in the header. It returns ObjectNotFound custom error if the object does not exists.

func (*S3) Size

func (s *S3) Size(ctx context.Context, path string) (int64, error)

Size returns the size of the content of the object. A retry mechanism is implemented because of the eventual consistency of S3 backends NotFound error are sometimes returned when the object was just uploaded.

func (*S3) Upload

func (s *S3) Upload(ctx context.Context, file io.Reader, path string) error

type S3Client

type S3Client interface {
	GetObject(ctx context.Context, input *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error)
	HeadObject(ctx context.Context, input *s3.HeadObjectInput, opts ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
	DeleteObject(ctx context.Context, input *s3.DeleteObjectInput, opts ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
}

type S3Config

type S3Config struct {
	AK       string
	SK       string
	Region   string
	Endpoint string
	Bucket   string
}

type Swift

type Swift struct {
	// contains filtered or unexported fields
}

func NewSwift

func NewSwift(cfg SwiftConfig) (*Swift, error)

NewSwift instantiate a new connection to a Swift object storage. The configuration is taken from the environment. Refer to the github.com/ncw/swift documentation for more information.

func (*Swift) Delete

func (s *Swift) Delete(ctx context.Context, path string) error

func (*Swift) Get

func (s *Swift) Get(ctx context.Context, path string) (io.ReadCloser, error)

func (*Swift) Info

func (s *Swift) Info(ctx context.Context, path string) (types.Info, error)

func (*Swift) Size

func (s *Swift) Size(ctx context.Context, path string) (int64, error)

Size returns the size of the content of the object. A retry mechanism is implemented because of the eventual consistency of Swift backends NotFound error are sometimes returned when the object was just uploaded.

func (*Swift) Upload

func (s *Swift) Upload(ctx context.Context, reader io.Reader, path string) error

type SwiftConfig

type SwiftConfig struct {
	Prefix    string
	Container string
	ChunkSize int64
}