storage

package
v0.0.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 24, 2020 License: Apache-2.0 Imports: 37 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	SUM_BLOCKSIZE              = 100 << 20 // 100M
	SUM_BENCHMARKTIMECONSUMING = 1         // 1s
	SUM_BASETIME               = 30        // 30s
)
View Source 
const (
	MAX_PART_SIZE   = 5 << 30 // 5GB
	MAX_PART_NUMBER = 10000
)
View Source 
const (
	RECYCLE_QUEUE_SIZE = 100
	MAX_TRY_TIMES      = 3
)
View Source 
const (
	AES_BLOCK_SIZE               = 16
	ENCRYPTION_KEY_LENGTH        = 32 // key size for AES-"256"
	INITIALIZATION_VECTOR_LENGTH = 16 // block size of AES
)
View Source 
const (
	BUCKET_NUMBER_LIMIT = 100
)
View Source 
const (
	CLUSTER_MAX_USED_SPACE_PERCENT = 85
)
View Source 
const (
	// only objects smaller than threshold are cached
	FILE_CACHE_THRESHOLD_SIZE = 4 << 20 // 4M
)

Variables

View Source 
var RecycleQueue chan objectToRecycle

Functions

func CheckBucketAclForGetObjectInfo 

func CheckBucketAclForGetObjectInfo(bucket *meta.Bucket, credential common.Credential) (err error)

func CheckObjectAclForGetObjectInfo 

func CheckObjectAclForGetObjectInfo(bucket *meta.Bucket, object *meta.Object, credential common.Credential) (err error)

func DeepCopyAcl 

func DeepCopyAcl(src, dst *datatype.Acl)

func DeepCopyAclPolicy 

func DeepCopyAclPolicy(src *datatype.AccessControlPolicy, dst *datatype.AccessControlPolicyResponse) error

Types

type DataCache 

type DataCache interface {
	WriteFromCache(object *meta.Object, startOffset int64, length int64,
		out io.Writer, writeThrough func(io.Writer) error,
		onCacheMiss func(io.Writer) error) error
	GetAlignedReader(object *meta.Object, startOffset int64, length int64,
		readThrough func() (io.ReadCloser, error),
		onCacheMiss func(io.Writer) error) (io.ReadCloser, error)
	Remove(key string)
}

type ReadCloser 

type ReadCloser struct {
	// contains filtered or unexported fields
}

func (*ReadCloser) Close 

func (r *ReadCloser) Close() error

func (*ReadCloser) Read 

func (r *ReadCloser) Read(b []byte) (n int, err error)

type YigStorage 

type YigStorage struct {
	DataStorage map[string]backend.Cluster
	DataCache   DataCache
	MetaStorage *meta.Meta
	KMS         crypto.KMS
	Stopping    bool
	WaitGroup   *sync.WaitGroup
}

*YigStorage implements api.ObjectLayer

func New 

func New(metaCacheType int, enableDataCache bool, kms crypto.KMS) *YigStorage

func (*YigStorage) AbortMultipartUpload 

func (yig *YigStorage) AbortMultipartUpload(reqCtx RequestContext, credential common.Credential, uploadId string) (deltaInfo datatype.DeltaSizeInfo, err error)

func (*YigStorage) AppendObject 

func (yig *YigStorage) AppendObject(reqCtx RequestContext, credential common.Credential,
	offset uint64, size int64, data io.ReadCloser, metadata map[string]string, acl Acl,
	sseRequest SseRequest, storageClass StorageClass, objInfo *meta.Object) (result AppendObjectResult, err error)

TODO: Append Support Encryption

func (*YigStorage) CheckBucketEncryption 

func (yig *YigStorage) CheckBucketEncryption(bucket *meta.Bucket) (*datatype.ApplyServerSideEncryptionByDefault, bool)

func (*YigStorage) CompleteMultipartUpload 

func (yig *YigStorage) CompleteMultipartUpload(reqCtx RequestContext, credential common.Credential, uploadId string, uploadedParts []meta.CompletePart) (result datatype.CompleteMultipartResult,
	err error)

func (*YigStorage) CopyObject 

func (yig *YigStorage) CopyObject(reqCtx RequestContext, targetObject *meta.Object, sourceObject *meta.Object, source io.Reader, credential common.Credential,
	sseRequest SseRequest, isMetadataOnly, isTranStorageClassOnly bool) (result PutObjectResult, err error)

func (*YigStorage) CopyObjectPart 

func (yig *YigStorage) CopyObjectPart(bucketName, objectName, uploadId string, partId int,
	size int64, data io.Reader, credential common.Credential,
	sseRequest datatype.SseRequest) (result datatype.PutObjectPartResult, err error)

func (*YigStorage) CreateFreezer 

func (yig *YigStorage) CreateFreezer(freezer *meta.Freezer) (err error)

func (*YigStorage) DelBucketLifecycle 

func (yig *YigStorage) DelBucketLifecycle(reqCtx RequestContext, credential common.Credential) error

func (*YigStorage) DeleteBucket 

func (yig *YigStorage) DeleteBucket(reqCtx RequestContext, credential common.Credential) (err error)

func (*YigStorage) DeleteBucketCors 

func (yig *YigStorage) DeleteBucketCors(reqCtx RequestContext, credential common.Credential) error

func (*YigStorage) DeleteBucketEncryption 

func (yig *YigStorage) DeleteBucketEncryption(bucket *meta.Bucket) error

func (*YigStorage) DeleteBucketPolicy 

func (yig *YigStorage) DeleteBucketPolicy(credential common.Credential, bucketName string) (err error)

func (*YigStorage) DeleteBucketWebsite 

func (yig *YigStorage) DeleteBucketWebsite(bucket *meta.Bucket) error

func (*YigStorage) DeleteObject 

func (yig *YigStorage) DeleteObject(reqCtx RequestContext,
	credential common.Credential) (result DeleteObjectResult, err error)

When bucket versioning is Disabled/Enabled/Suspended, and request versionId is set/unset:

| | with versionId | without versionId | |-----------|------------------------------|--------------------------------------------------------| | Disabled | error | remove object | | Enabled | remove corresponding version | add a delete marker | | Suspended | remove corresponding version | remove null version object(if exists) and add a | | | | null version delete marker |

See http://docs.aws.amazon.com/AmazonS3/latest/dev/Versioning.html

func (*YigStorage) DeleteObjects 

func (yig *YigStorage) DeleteObjects(reqCtx RequestContext, credential common.Credential,
	objects []ObjectIdentifier) (result DeleteObjectsResult, err error)

func (*YigStorage) EliminateObject 

func (yig *YigStorage) EliminateObject(freezer *meta.Freezer) (err error)

func (*YigStorage) GetBucket 

func (yig *YigStorage) GetBucket(bucketName string) (*meta.Bucket, error)

For INTERNAL USE ONLY

func (*YigStorage) GetBucketAcl 

func (yig *YigStorage) GetBucketAcl(reqCtx RequestContext, credential common.Credential) (
	policy datatype.AccessControlPolicyResponse, err error)

func (*YigStorage) GetBucketCors 

func (yig *YigStorage) GetBucketCors(reqCtx RequestContext,
	credential common.Credential) (cors datatype.Cors, err error)

func (*YigStorage) GetBucketEncryption 

func (yig *YigStorage) GetBucketEncryption(bucketName string) (config datatype.EncryptionConfiguration, err error)

func (*YigStorage) GetBucketInfo 

func (yig *YigStorage) GetBucketInfo(reqCtx RequestContext,
	credential common.Credential) (bucket *meta.Bucket, err error)

func (*YigStorage) GetBucketInfoByCtx 

func (yig *YigStorage) GetBucketInfoByCtx(reqCtx RequestContext,
	credential common.Credential) (bucket *meta.Bucket, err error)

func (*YigStorage) GetBucketLifecycle 

func (yig *YigStorage) GetBucketLifecycle(reqCtx RequestContext, credential common.Credential) (lc lifecycle.Lifecycle,
	err error)

func (*YigStorage) GetBucketLogging 

func (yig *YigStorage) GetBucketLogging(reqCtx RequestContext, credential common.Credential) (logging datatype.BucketLoggingStatus,
	err error)

func (*YigStorage) GetBucketPolicy 

func (yig *YigStorage) GetBucketPolicy(credential common.Credential, bucketName string) (bucketPolicy policy.Policy, err error)

func (*YigStorage) GetBucketVersioning 

func (yig *YigStorage) GetBucketVersioning(reqCtx RequestContext, credential common.Credential) (
	versioning datatype.Versioning, err error)

func (*YigStorage) GetBucketWebsite 

func (yig *YigStorage) GetBucketWebsite(bucketName string) (config datatype.WebsiteConfiguration, err error)

func (*YigStorage) GetClusterByFsName 

func (yig *YigStorage) GetClusterByFsName(fsName string) (cluster backend.Cluster, err error)

func (*YigStorage) GetFreezer 

func (yig *YigStorage) GetFreezer(bucketName string, objectName string, version string) (freezer *meta.Freezer, err error)

func (*YigStorage) GetFreezerStatus 

func (yig *YigStorage) GetFreezerStatus(bucketName string, objectName string, version string) (freezer *meta.Freezer, err error)

func (*YigStorage) GetObject 

func (yig *YigStorage) GetObject(object *meta.Object, startOffset int64,
	length int64, writer io.Writer, sseRequest SseRequest) (err error)

func (*YigStorage) GetObjectAcl 

func (yig *YigStorage) GetObjectAcl(reqCtx RequestContext, credential common.Credential) (
	policy AccessControlPolicyResponse, err error)

func (*YigStorage) GetObjectInfo 

func (yig *YigStorage) GetObjectInfo(bucketName string, objectName string,
	version string, credential common.Credential) (object *meta.Object, err error)

func (*YigStorage) GetObjectInfoByCtx 

func (yig *YigStorage) GetObjectInfoByCtx(ctx RequestContext, credential common.Credential) (object *meta.Object, err error)

func (*YigStorage) ListBuckets 

func (yig *YigStorage) ListBuckets(credential common.Credential) (buckets []meta.Bucket, err error)

func (*YigStorage) ListMultipartUploads 

func (yig *YigStorage) ListMultipartUploads(reqCtx RequestContext, credential common.Credential,
	request datatype.ListUploadsRequest) (result datatype.ListMultipartUploadsResponse, err error)

func (*YigStorage) ListObjectParts 

func (yig *YigStorage) ListObjectParts(credential common.Credential, bucketName, objectName string,
	request datatype.ListPartsRequest) (result datatype.ListPartsResponse, err error)

func (*YigStorage) ListObjects 

func (yig *YigStorage) ListObjects(reqCtx RequestContext, credential common.Credential,
	request datatype.ListObjectsRequest) (result meta.ListObjectsInfo, err error)

func (*YigStorage) ListObjectsInternal 

func (yig *YigStorage) ListObjectsInternal(bucket *meta.Bucket, request datatype.ListObjectsRequest) (
	info meta.ListObjectsInfo, err error)

func (*YigStorage) ListVersionedObjects 

func (yig *YigStorage) ListVersionedObjects(reqCtx RequestContext, credential common.Credential,
	request datatype.ListObjectsRequest) (result meta.VersionedListObjectsInfo, err error)

TODO: refactor, similar to ListObjects or not?

func (*YigStorage) ListVersionedObjectsInternal 

func (yig *YigStorage) ListVersionedObjectsInternal(bucketName string,
	request datatype.ListObjectsRequest) (info meta.VersionedListObjectsInfo, err error)

func (*YigStorage) MakeBucket 

func (yig *YigStorage) MakeBucket(reqCtx RequestContext, acl datatype.Acl,
	credential common.Credential) error

func (*YigStorage) NewMultipartUpload 

func (yig *YigStorage) NewMultipartUpload(reqCtx RequestContext, credential common.Credential,
	metadata map[string]string, acl datatype.Acl,
	sseRequest datatype.SseRequest, storageClass StorageClass) (uploadId string, err error)

func (*YigStorage) PingCache 

func (y *YigStorage) PingCache(interval time.Duration)

check cache health per one second if enable cache

func (*YigStorage) PutObject 

func (yig *YigStorage) PutObject(reqCtx RequestContext, credential common.Credential,
	size int64, data io.ReadCloser, metadata map[string]string, acl Acl,
	sseRequest SseRequest, storageClass StorageClass) (result PutObjectResult, err error)

Write path: 

+-----------+

PUT object/part | | Ceph 

+---------+------------+----------+ Encryptor +----->
          |            |          |           |
          |            |          +-----------+
          v            v
         SHA256      MD5(ETag)

SHA256 is calculated only for v4 signed authentication Encryptor is enabled when user set SSE headers

func (*YigStorage) PutObjectMeta 

func (yig *YigStorage) PutObjectMeta(bucket *meta.Bucket, targetObject *meta.Object, credential common.Credential) (err error)

func (*YigStorage) PutObjectPart 

func (yig *YigStorage) PutObjectPart(reqCtx RequestContext, credential common.Credential,
	uploadId string, partId int, size int64, data io.ReadCloser, md5Hex string,
	sseRequest datatype.SseRequest) (result datatype.PutObjectPartResult, err error)

func (*YigStorage) RenameObject 

func (yig *YigStorage) RenameObject(reqCtx RequestContext, targetObject *meta.Object, sourceObject string, credential common.Credential) (result RenameObjectResult, err error)

func (*YigStorage) RestoreObject 

func (yig *YigStorage) RestoreObject(freezer *meta.Freezer) (err error)

func (*YigStorage) SetBucketAcl 

func (yig *YigStorage) SetBucketAcl(reqCtx RequestContext, acl datatype.Acl,
	credential common.Credential) error

func (*YigStorage) SetBucketCors 

func (yig *YigStorage) SetBucketCors(reqCtx RequestContext, cors datatype.Cors,
	credential common.Credential) error

func (*YigStorage) SetBucketEncryption 

func (yig *YigStorage) SetBucketEncryption(bucket *meta.Bucket, config datatype.EncryptionConfiguration) (err error)

func (*YigStorage) SetBucketLifecycle 

func (yig *YigStorage) SetBucketLifecycle(reqCtx RequestContext, lc lifecycle.Lifecycle,
	credential common.Credential) error

func (*YigStorage) SetBucketLogging 

func (yig *YigStorage) SetBucketLogging(reqCtx RequestContext, bl datatype.BucketLoggingStatus,
	credential common.Credential) error

func (*YigStorage) SetBucketPolicy 

func (yig *YigStorage) SetBucketPolicy(credential common.Credential, bucketName string, bucketPolicy policy.Policy) (err error)

func (*YigStorage) SetBucketVersioning 

func (yig *YigStorage) SetBucketVersioning(reqCtx RequestContext, versioning datatype.Versioning,
	credential common.Credential) error

func (*YigStorage) SetBucketWebsite 

func (yig *YigStorage) SetBucketWebsite(bucket *meta.Bucket, config datatype.WebsiteConfiguration) (err error)

func (*YigStorage) SetObjectAcl 

func (yig *YigStorage) SetObjectAcl(reqCtx RequestContext, acl Acl,
	credential common.Credential) error

func (*YigStorage) Stop 

func (y *YigStorage) Stop()

func (*YigStorage) UpdateFreezerDate 

func (yig *YigStorage) UpdateFreezerDate(freezer *meta.Freezer, date int, isIncrement bool) (err error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.