storage

package
v1.6.8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 30, 2024 License: GPL-3.0 Imports: 40 Imported by: 0

Documentation

Overview

Package storage is a generated GoMock package.

Index

Constants

View Source 
const (
	// S3Store defines storage type for s3
	S3Store = "s3"
	// OSSStore defines storage type for oss
	OSSStore = "oss"
	// B2Store defines storage type for s3
	B2Store = "b2"
	// MinioStore defines storage type for minio
	MinioStore = "minio"
	// DiskFileStore defines storage type for file
	DiskFileStore = "file"
	// LdfsStore defines storage type for ldfs
	LdfsStore = "ldfs"
	// MemoryStore defines storage type for memory
	MemoryStore = "memory"
)

define storage type constants

View Source 
const (
	// AKSKIAMType defines IAM type config which uses access key and secret key to access aws s3
	AKSKIAMType = "AKSK"
	// SAIAMType defines IAM type config which uses service account to access aws s3
	SAIAMType = "SA"

	// AWSRoleARN defines env variable for aws role arn
	AWSRoleARN = "AWS_ROLE_ARN"
	// AWSWebIdentityTokenFile defines env variable for aws identity token file
	AWSWebIdentityTokenFile = "AWS_WEB_IDENTITY_TOKEN_FILE"

	// BucketURL defines env variable name for bucket url
	BucketURL = "BUCKET_URL"
	// AWSAccessKey defines env variable name for aws access key
	AWSAccessKey = "AWS_ACCESS_KEY"
	// AWSSecretKey defines env variable name for aws secret key
	AWSSecretKey = "AWS_SECRET_KEY"
	// AWSSessionToken defines env variable name for aws session token
	AWSSessionToken = "AWS_SESSION_TOKEN"

	// OSSRoleARN defines env variable for OSS role arn
	OSSRoleARN = "ALIBABA_CLOUD_ROLE_ARN"
	// OSSWebIdentityTokenFile defines env variable for OSS identity token file
	OSSWebIdentityTokenFile = "ALIBABA_CLOUD_OIDC_TOKEN_FILE"
	// OSSOidcProviderArn defines env variable for OSS oidc provider arn
	OSSOidcProviderArn = "ALIBABA_CLOUD_OIDC_PROVIDER_ARN"
	// OSSAccessKey defines env variable name for OSS access key
	OSSAccessKey = "ALIBABA_CLOUD_ACCESS_KEY"
	// OSSSecretKey defines env variable name for OSS secret key
	OSSSecretKey = "ALIBABA_CLOUD_SECRET_KEY"
	// OSSSessionToken defines env variable name for OSS session token
	OSSSessionToken = "ALIBABA_CLOUD_SESSION_TOKEN"
	// OSSRegion defines env variable name for OSS oss region
	OSSRegion = "ALIBABA_CLOUD_OSS_REGION"

	// MinioRegion defines env variable name for minio region
	MinioRegion = "MINIO_REGION"
	// MinioAccessKey defines env variable name for minio access key
	MinioAccessKey = "MINIO_ACCESS_KEY"
	// MinioSecretKey defines env variable name for minio secret key
	MinioSecretKey = "MINIO_SECRET_KEY"
	// MinioSessionToken defines env variable name for minio session token
	MinioSessionToken = "MINIO_SESSION_TOKEN"

	// B2AccessKey defines env variable name for minio access key
	B2AccessKey = "B2_ACCESS_KEY"
	// B2SecretKey defines env variable name for minio secret key
	B2SecretKey = "B2_SECRET_KEY"
	// B2SessionToken defines env variable name for minio session token
	B2SessionToken = "B2_SESSION_TOKEN"

	// OctetStream is used to indicate the binary files
	OctetStream = "application/octet-stream"
)

piece store storage config and environment constants

View Source 
const (
	UserAgent = "Greenfield-SP"
	// BufPoolSize define buffer pool size
	BufPoolSize = 32 << 10
	// ChecksumAlgo define validation algorithm name
	ChecksumAlgo = "Crc32c"
)

define piece store constants.

Variables

View Source 
var (
	// ErrNoSuchObject defines not existed object error
	ErrNoSuchObject = errors.New("the specified key does not exist")
	// ErrNoSuchBucket defines not existed bucket error
	ErrNoSuchBucket = errors.New("the specified bucket does not exist")
	// ErrUnsupportedDelimiter defines invalid key with delimiter error
	ErrUnsupportedDelimiter = errors.New("unsupported delimiter")
	// ErrInvalidObjectKey defines invalid object key error
	ErrInvalidObjectKey = errors.New("invalid object key")
	// ErrUnsupportedMethod defines unsupported method error
	ErrUnsupportedMethod = errors.New("unsupported method")
	// ErrNoPermissionAccessBucket defines deny access bucket error
	ErrNoPermissionAccessBucket = errors.New("deny access bucket")
)

piece store errors

Functions

This section is empty.

Types

type DefaultObjectStorage 

type DefaultObjectStorage struct{}

func (DefaultObjectStorage) CreateBucket 

func (s DefaultObjectStorage) CreateBucket(ctx context.Context) error

func (DefaultObjectStorage) ListAllObjects 

func (s DefaultObjectStorage) ListAllObjects(ctx context.Context, prefix, marker string) (<-chan Object, error)

func (DefaultObjectStorage) ListObjects 

func (s DefaultObjectStorage) ListObjects(ctx context.Context, prefix, marker, delimiter string, limit int64) ([]Object, error)

type MockObject added in v0.2.4 

type MockObject struct {
	// contains filtered or unexported fields
}

MockObject is a mock of Object interface.

func NewMockObject added in v0.2.4 

func NewMockObject(ctrl *gomock.Controller) *MockObject

NewMockObject creates a new mock instance.

func (*MockObject) EXPECT added in v0.2.4 

func (m *MockObject) EXPECT() *MockObjectMockRecorder

EXPECT returns an object that allows the caller to indicate expected use. 

func (m *MockObject) IsSymlink() bool

IsSymlink mocks base method.

func (*MockObject) Key added in v0.2.4 

func (m *MockObject) Key() string

Key mocks base method.

func (*MockObject) ModTime added in v0.2.4 

func (m *MockObject) ModTime() time.Time

ModTime mocks base method.

func (*MockObject) Size added in v0.2.4 

func (m *MockObject) Size() int64

Size mocks base method.

type MockObjectMockRecorder added in v0.2.4 

type MockObjectMockRecorder struct {
	// contains filtered or unexported fields
}

MockObjectMockRecorder is the mock recorder for MockObject. 

func (mr *MockObjectMockRecorder) IsSymlink() *gomock.Call

IsSymlink indicates an expected call of IsSymlink.

func (*MockObjectMockRecorder) Key added in v0.2.4 

func (mr *MockObjectMockRecorder) Key() *gomock.Call

Key indicates an expected call of Key.

func (*MockObjectMockRecorder) ModTime added in v0.2.4 

func (mr *MockObjectMockRecorder) ModTime() *gomock.Call

ModTime indicates an expected call of ModTime.

func (*MockObjectMockRecorder) Size added in v0.2.4 

func (mr *MockObjectMockRecorder) Size() *gomock.Call

Size indicates an expected call of Size.

type MockObjectStorage added in v0.2.4 

type MockObjectStorage struct {
	// contains filtered or unexported fields
}

MockObjectStorage is a mock of ObjectStorage interface.

func NewMockObjectStorage added in v0.2.4 

func NewMockObjectStorage(ctrl *gomock.Controller) *MockObjectStorage

NewMockObjectStorage creates a new mock instance.

func (*MockObjectStorage) CreateBucket added in v0.2.4 

func (m *MockObjectStorage) CreateBucket(ctx context.Context) error

CreateBucket mocks base method.

func (*MockObjectStorage) DeleteObject added in v0.2.4 

func (m *MockObjectStorage) DeleteObject(ctx context.Context, key string) error

DeleteObject mocks base method.

func (*MockObjectStorage) EXPECT added in v0.2.4 

func (m *MockObjectStorage) EXPECT() *MockObjectStorageMockRecorder

EXPECT returns an object that allows the caller to indicate expected use.

func (*MockObjectStorage) GetObject added in v0.2.4 

func (m *MockObjectStorage) GetObject(ctx context.Context, key string, offset, limit int64) (io.ReadCloser, error)

GetObject mocks base method.

func (*MockObjectStorage) HeadBucket added in v0.2.4 

func (m *MockObjectStorage) HeadBucket(ctx context.Context) error

HeadBucket mocks base method.

func (*MockObjectStorage) HeadObject added in v0.2.4 

func (m *MockObjectStorage) HeadObject(ctx context.Context, key string) (Object, error)

HeadObject mocks base method.

func (*MockObjectStorage) ListAllObjects added in v0.2.4 

func (m *MockObjectStorage) ListAllObjects(ctx context.Context, prefix, marker string) (<-chan Object, error)

ListAllObjects mocks base method.

func (*MockObjectStorage) ListObjects added in v0.2.4 

func (m *MockObjectStorage) ListObjects(ctx context.Context, prefix, marker, delimiter string, limit int64) ([]Object, error)

ListObjects mocks base method.

func (*MockObjectStorage) PutObject added in v0.2.4 

func (m *MockObjectStorage) PutObject(ctx context.Context, key string, reader io.Reader) error

PutObject mocks base method.

func (*MockObjectStorage) String added in v0.2.4 

func (m *MockObjectStorage) String() string

String mocks base method.

type MockObjectStorageMockRecorder added in v0.2.4 

type MockObjectStorageMockRecorder struct {
	// contains filtered or unexported fields
}

MockObjectStorageMockRecorder is the mock recorder for MockObjectStorage.

func (*MockObjectStorageMockRecorder) CreateBucket added in v0.2.4 

func (mr *MockObjectStorageMockRecorder) CreateBucket(ctx any) *gomock.Call

CreateBucket indicates an expected call of CreateBucket.

func (*MockObjectStorageMockRecorder) DeleteObject added in v0.2.4 

func (mr *MockObjectStorageMockRecorder) DeleteObject(ctx, key any) *gomock.Call

DeleteObject indicates an expected call of DeleteObject.

func (*MockObjectStorageMockRecorder) GetObject added in v0.2.4 

func (mr *MockObjectStorageMockRecorder) GetObject(ctx, key, offset, limit any) *gomock.Call

GetObject indicates an expected call of GetObject.

func (*MockObjectStorageMockRecorder) HeadBucket added in v0.2.4 

func (mr *MockObjectStorageMockRecorder) HeadBucket(ctx any) *gomock.Call

HeadBucket indicates an expected call of HeadBucket.

func (*MockObjectStorageMockRecorder) HeadObject added in v0.2.4 

func (mr *MockObjectStorageMockRecorder) HeadObject(ctx, key any) *gomock.Call

HeadObject indicates an expected call of HeadObject.

func (*MockObjectStorageMockRecorder) ListAllObjects added in v0.2.4 

func (mr *MockObjectStorageMockRecorder) ListAllObjects(ctx, prefix, marker any) *gomock.Call

ListAllObjects indicates an expected call of ListAllObjects.

func (*MockObjectStorageMockRecorder) ListObjects added in v0.2.4 

func (mr *MockObjectStorageMockRecorder) ListObjects(ctx, prefix, marker, delimiter, limit any) *gomock.Call

ListObjects indicates an expected call of ListObjects.

func (*MockObjectStorageMockRecorder) PutObject added in v0.2.4 

func (mr *MockObjectStorageMockRecorder) PutObject(ctx, key, reader any) *gomock.Call

PutObject indicates an expected call of PutObject.

func (*MockObjectStorageMockRecorder) String added in v0.2.4 

func (mr *MockObjectStorageMockRecorder) String() *gomock.Call

String indicates an expected call of String.

type Object 

type Object interface {
	Key() string
	Size() int64
	ModTime() time.Time
	IsSymlink() bool
}

Object

type ObjectStorage 

type ObjectStorage interface {
	// String the description of an object storage
	String() string
	// CreateBucket create the bucket if not existed
	CreateBucket(ctx context.Context) error
	// GetObject gets data for the given object specified by key
	GetObject(ctx context.Context, key string, offset, limit int64) (io.ReadCloser, error)
	// PutObject puts data read from a reader to an object specified by key
	PutObject(ctx context.Context, key string, reader io.Reader) error
	// DeleteObject deletes an object
	DeleteObject(ctx context.Context, key string) error

	// HeadBucket determines if a bucket exists and have permission to access it
	HeadBucket(ctx context.Context) error
	// HeadObject returns some information about the object or an error if not found
	HeadObject(ctx context.Context, key string) (Object, error)
	// ListObjects lists returns a list of objects
	ListObjects(ctx context.Context, prefix, marker, delimiter string, limit int64) ([]Object, error)
	// ListAllObjects returns all the objects as a channel
	ListAllObjects(ctx context.Context, prefix, marker string) (<-chan Object, error)
}

ObjectStorage is a common interface that must be implemented if some users want to use an object storage (such as S3, Azure Blob, Minio, OSS, COS, etc)

func NewObjectStorage 

func NewObjectStorage(cfg ObjectStorageConfig) (ObjectStorage, error)

func NewSharded 

func NewSharded(cfg PieceStoreConfig) (ObjectStorage, error)

type ObjectStorageConfig 

type ObjectStorageConfig struct {
	// Storage backend storage type (e.g. s3, file, memory)
	Storage string `comment:"required"`
	// BucketURL the bucket URL of object storage to store data
	BucketURL string `comment:"optional"`
	// MaxRetries the number of max retries that will be performed
	MaxRetries int `comment:"optional"`
	// MinRetryDelay the minimum retry delay after which retry will be performed
	MinRetryDelay int64 `comment:"optional"`
	// TLSInsecureSkipVerify whether skip the certificate verification of HTTPS requests
	TLSInsecureSkipVerify bool `comment:"optional"`
	// IAMType is identity and access management type which contains two types: AKSKIAMType/SAIAMType
	IAMType string `comment:"required"`
}

ObjectStorageConfig object storage config

type PieceStoreConfig 

type PieceStoreConfig struct {
	// Shards store the blocks into N buckets by hash of key
	Shards int `comment:"required"`
	// Store config of object storage
	Store ObjectStorageConfig
}

PieceStoreConfig contains some parameters which are used to run PieceStore

type SessionCache 

type SessionCache struct {
	sync.Mutex
	// contains filtered or unexported fields
}

SessionCache holds session.Session according to ObjectStorageConfig and it synchronizes access/modification

type StorageFn 

type StorageFn func(cfg ObjectStorageConfig) (ObjectStorage, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.