storage

Documentation

Index

• Constants
• Variables
• type DefaultObjectStorage
  • func (s DefaultObjectStorage) CreateBucket(ctx context.Context) error
  • func (s DefaultObjectStorage) ListAllObjects(ctx context.Context, prefix, marker string) (<-chan Object, error)
  • func (s DefaultObjectStorage) ListObjects(ctx context.Context, prefix, marker, delimiter string, limit int64) ([]Object, error)
• type Object
• type ObjectStorage
  • func NewObjectStorage(cfg ObjectStorageConfig) (ObjectStorage, error)
  • func NewSharded(cfg PieceStoreConfig) (ObjectStorage, error)
• type ObjectStorageConfig
• type PieceStoreConfig
• type SessionCache
• type StorageFn

Constants

const (
	// S3Store defines storage type for s3
	S3Store = "s3"
	// B2Store defines storage type for s3
	B2Store = "b2"
	// MinioStore defines storage type for minio
	MinioStore = "minio"
	// DiskFileStore defines storage type for file
	DiskFileStore = "file"
	// LdfsStore defines storage type for ldfs
	LdfsStore = "ldfs"
	// MemoryStore defines storage type for memory
	MemoryStore = "memory"
)

define storage type constants

const (
	// AKSKIAMType defines IAM type config which uses access key and secret key to access aws s3
	AKSKIAMType = "AKSK"
	// SAIAMType defines IAM type config which uses service account to access aws s3
	SAIAMType = "SA"

	// AWSRoleARN defines env variable for aws role arn
	AWSRoleARN = "AWS_ROLE_ARN"
	// AWSWebIdentityTokenFile defines env variable for aws identity token file
	AWSWebIdentityTokenFile = "AWS_WEB_IDENTITY_TOKEN_FILE"

	// BucketURL defines env variable name for bucket url
	BucketURL = "BUCKET_URL"
	// AWSAccessKey defines env variable name for aws access key
	AWSAccessKey = "AWS_ACCESS_KEY"
	// AWSSecretKey defines env variable name for aws secret key
	AWSSecretKey = "AWS_SECRET_KEY"
	// AWSSessionToken defines env variable name for aws session token
	AWSSessionToken = "AWS_SESSION_TOKEN"

	// MinioRegion defines env variable name for minio region
	MinioRegion = "MINIO_REGION"
	// MinioAccessKey defines env variable name for minio access key
	MinioAccessKey = "MINIO_ACCESS_KEY"
	// MinioSecretKey defines env variable name for minio secret key
	MinioSecretKey = "MINIO_SECRET_KEY"
	// MinioSessionToken defines env variable name for minio session token
	MinioSessionToken = "MINIO_SESSION_TOKEN"

	// B2AccessKey defines env variable name for minio access key
	B2AccessKey = "B2_ACCESS_KEY"
	// B2SecretKey defines env variable name for minio secret key
	B2SecretKey = "B2_SECRET_KEY"
	// B2SessionToken defines env variable name for minio session token
	B2SessionToken = "B2_SESSION_TOKEN"

	// OctetStream is used to indicate the binary files
	OctetStream = "application/octet-stream"
)

piece store storage config and environment constants

const (
	// BufPoolSize define buffer pool size
	BufPoolSize = 32 << 10
	// ChecksumAlgo define validation algorithm name
	ChecksumAlgo = "Crc32c"
)

define piece store constants.

Variables

var (
	// ErrNoSuchObject defines not existed object error
	ErrNoSuchObject = errors.New("the specified key does not exist")
	// ErrNoSuchBucket defines not existed bucket error
	ErrNoSuchBucket = errors.New("the specified bucket does not exist")
	// ErrUnsupportedDelimiter defines invalid key with delimiter error
	ErrUnsupportedDelimiter = errors.New("unsupported delimiter")
	// ErrInvalidObjectKey defines invalid object key error
	ErrInvalidObjectKey = errors.New("invalid object key")
	// ErrUnsupportedMethod defines unsupported method error
	ErrUnsupportedMethod = errors.New("unsupported method")
	// ErrNoPermissionAccessBucket defines deny access bucket error
	ErrNoPermissionAccessBucket = errors.New("deny access bucket")
)

piece store errors

Types

type DefaultObjectStorage

type DefaultObjectStorage struct{}

func (DefaultObjectStorage) CreateBucket

func (s DefaultObjectStorage) CreateBucket(ctx context.Context) error

func (DefaultObjectStorage) ListAllObjects

func (s DefaultObjectStorage) ListAllObjects(ctx context.Context, prefix, marker string) (<-chan Object, error)

func (DefaultObjectStorage) ListObjects

func (s DefaultObjectStorage) ListObjects(ctx context.Context, prefix, marker, delimiter string, limit int64) ([]Object, error)

type Object

type Object interface {
	Key() string
	Size() int64
	ModTime() time.Time
	IsSymlink() bool
}

Object

type ObjectStorage

type ObjectStorage interface {
	// String the description of an object storage
	String() string
	// CreateBucket create the bucket if not existed
	CreateBucket(ctx context.Context) error
	// GetObject gets data for the given object specified by key
	GetObject(ctx context.Context, key string, offset, limit int64) (io.ReadCloser, error)
	// PutObject puts data read from a reader to an object specified by key
	PutObject(ctx context.Context, key string, reader io.Reader) error
	// DeleteObject deletes an object
	DeleteObject(ctx context.Context, key string) error

	// HeadBucket determines if a bucket exists and have permission to access it
	HeadBucket(ctx context.Context) error
	// HeadObject returns some information about the object or an error if not found
	HeadObject(ctx context.Context, key string) (Object, error)
	// ListObjects lists returns a list of objects
	ListObjects(ctx context.Context, prefix, marker, delimiter string, limit int64) ([]Object, error)
	// ListAllObjects returns all the objects as a channel
	ListAllObjects(ctx context.Context, prefix, marker string) (<-chan Object, error)
}

ObjectStorage is a common interface that must be implemented if some users want to use an object storage (such as S3, Azure Blob, Minio, OSS, COS, etc)

func NewObjectStorage

func NewObjectStorage(cfg ObjectStorageConfig) (ObjectStorage, error)

func NewSharded

func NewSharded(cfg PieceStoreConfig) (ObjectStorage, error)

type ObjectStorageConfig

type ObjectStorageConfig struct {
	Storage               string // backend storage type (e.g. s3, file, memory)
	BucketURL             string // the bucket URL of object storage to store data
	MaxRetries            int    // the number of max retries that will be performed
	MinRetryDelay         int64  // the minimum retry delay after which retry will be performed
	TLSInsecureSkipVerify bool   // whether skip the certificate verification of HTTPS requests
	IAMType               string // IAMType is identity and access management type which contains two types: AKSKIAMType/SAIAMType
}

ObjectStorageConfig object storage config

type PieceStoreConfig

type PieceStoreConfig struct {
	Shards int                 // store the blocks into N buckets by hash of key
	Store  ObjectStorageConfig // config of object storage
}

PieceStoreConfig contains some parameters which are used to run PieceStore

type SessionCache

type SessionCache struct {
	sync.Mutex
	// contains filtered or unexported fields
}

SessionCache holds session.Session according to ObjectStorageConfig and it synchronizes access/modification

type StorageFn

type StorageFn func(cfg ObjectStorageConfig) (ObjectStorage, error)