meta

README

Meta layer

Storage metadata manipulation helper functions for YIG, including a cache layer for database, build on Redis.

Move more methods into this package as more patterns emerge

Documentation

Index

• type CacheType
• type Meta
  • func New(myCacheType CacheType) *Meta
  • func (m *Meta) AbortTrans(tx Tx) (err error)
  • func (m *Meta) AddDeleteMarker(marker *Object) (err error)
  • func (m *Meta) AppendObject(object *Object, olderObject *Object) error
  • func (m *Meta) CommitTrans(tx Tx) (err error)
  • func (m *Meta) CreateFreezer(freezer *types.Freezer) error
  • func (m *Meta) DeleteFreezer(freezer *types.Freezer) (err error)
  • func (m *Meta) DeleteFreezerWithoutCephObject(bucketName, objectName, version string, freezerType types.ObjectType, ...) (err error)
  • func (m *Meta) DeleteMultipart(multipart Multipart) (removedSize int64, err error)
  • func (m *Meta) DeleteObject(object *Object) (err error)
  • func (m *Meta) DeleteObjectWithTx(object *Object, tx Tx) (err error)
  • func (m *Meta) DeleteSuspendedObject(object *Object) (err error)
  • func (m *Meta) GetBucket(bucketName string, willNeed bool) (bucket *Bucket, err error)
  • func (m *Meta) GetBucketInfo(bucketName string) (*Bucket, error)
  • func (m *Meta) GetBucketLifeCycle(bucket Bucket) (lifeCycle *LifeCycle, err error)
  • func (m *Meta) GetBuckets() (buckets []Bucket, err error)
  • func (m *Meta) GetClusters() (cluster []Cluster, err error)
  • func (m *Meta) GetFreezer(bucketName string, objectName string, version string) (freezer *types.Freezer, err error)
  • func (m *Meta) GetFreezerStatus(bucketName string, objectName string, version string) (freezer *types.Freezer, err error)
  • func (m *Meta) GetMultipart(bucketName, objectName, uploadId string) (multipart Multipart, err error)
  • func (m *Meta) GetObject(bucketName, objectName, reqVersion string, willNeed bool) (object *Object, err error)
  • func (m *Meta) GetUsage(bucketName string) (int64, error)
  • func (m *Meta) GetUserBuckets(userId string, willNeed bool) (buckets []string, err error)
  • func (m *Meta) GetUserInfo(userId string) ([]string, error)
  • func (m *Meta) MigrateObject(object *Object) error
  • func (m *Meta) NewTrans() (tx Tx, err error)
  • func (m *Meta) PutBucketToLifeCycle(bucket Bucket) error
  • func (m *Meta) PutFreezer(freezer *types.Freezer, status common.RestoreStatus) error
  • func (m *Meta) PutFreezerToGarbageCollection(freezer *Freezer) error
  • func (m *Meta) PutObject(reqCtx RequestContext, object *Object, multipart *Multipart, updateUsage bool) (deltaInfo map[common.StorageClass]int64, err error)
  • func (m *Meta) PutObjectPart(multipart Multipart, part Part) (deltaSize int64, err error)
  • func (m *Meta) PutObjectToGarbageCollection(object *Object) error
  • func (m *Meta) RemoveBucketFromLifeCycle(bucket Bucket) error
  • func (m *Meta) RemoveHotObject(object *Object, tx Tx) error
  • func (m *Meta) RenameObject(object *Object, sourceObject string) error
  • func (m *Meta) ReplaceObjectMetas(object *Object) error
  • func (m *Meta) ScanLifeCycle(limit int, marker string) (result ScanLifeCycleResult, err error)
  • func (m *Meta) UpdateFreezerDate(freezer *types.Freezer) error
  • func (m *Meta) UpdateGlacierObject(reqCtx RequestContext, targetObject, sourceObject *Object) (err error)
  • func (m *Meta) UpdateObjectAcl(object *Object) error
  • func (m *Meta) UpdateObjectAttrs(object *Object) error
• type MetaCache
• type QosMeta
  • func NewQosMeta(client client.Client) *QosMeta
  • func (m *QosMeta) AllowReadQuery(bucketName string) (allow bool)
  • func (m *QosMeta) AllowWriteQuery(bucketName string) (allow bool)
  • func (m *QosMeta) NewThrottleReader(bucketName string, reader io.Reader) *ThrottleReader
  • func (m *QosMeta) NewThrottleWriter(bucketName string, writer io.Writer) *ThrottleWriter
• type ThrottleReader
  • func (r *ThrottleReader) Close()
  • func (r *ThrottleReader) Read(p []byte) (int, error)
• type ThrottleWriter
  • func (w *ThrottleWriter) Close()
  • func (w *ThrottleWriter) Write(p []byte) (int, error)

Types

type CacheType

type CacheType int

const (
	NoCache CacheType = iota
	EnableCache
	SimpleCache
)

type Meta

type Meta struct {
	Client                    client.Client
	Cache                     MetaCache
	QosMeta                   *QosMeta
	GarbageCollectionProducer kafka.Producer
	// contains filtered or unexported fields
}

func New

func New(myCacheType CacheType) *Meta

func (*Meta) AbortTrans

func (m *Meta) AbortTrans(tx Tx) (err error)

func (*Meta) AddDeleteMarker

func (m *Meta) AddDeleteMarker(marker *Object) (err error)

func (*Meta) AppendObject

func (m *Meta) AppendObject(object *Object, olderObject *Object) error

func (*Meta) CommitTrans

func (m *Meta) CommitTrans(tx Tx) (err error)

func (*Meta) CreateFreezer

func (m *Meta) CreateFreezer(freezer *types.Freezer) error

func (*Meta) DeleteFreezer

func (m *Meta) DeleteFreezer(freezer *types.Freezer) (err error)

func (*Meta) DeleteFreezerWithoutCephObject

func (m *Meta) DeleteFreezerWithoutCephObject(bucketName, objectName, version string, freezerType types.ObjectType, createTime uint64) (err error)

func (*Meta) DeleteMultipart

func (m *Meta) DeleteMultipart(multipart Multipart) (removedSize int64, err error)

func (*Meta) DeleteObject

func (m *Meta) DeleteObject(object *Object) (err error)

func (*Meta) DeleteObjectWithTx

func (m *Meta) DeleteObjectWithTx(object *Object, tx Tx) (err error)

func (*Meta) DeleteSuspendedObject

func (m *Meta) DeleteSuspendedObject(object *Object) (err error)

func (*Meta) GetBucket

func (m *Meta) GetBucket(bucketName string, willNeed bool) (bucket *Bucket, err error)

Note the usage info got from this method is possibly not accurate because we don't invalid cache when updating usage. For accurate usage info, use `GetUsage()`

func (*Meta) GetBucketInfo

func (m *Meta) GetBucketInfo(bucketName string) (*Bucket, error)

func (*Meta) GetBucketLifeCycle

func (m *Meta) GetBucketLifeCycle(bucket Bucket) (lifeCycle *LifeCycle, err error)

func (*Meta) GetBuckets

func (m *Meta) GetBuckets() (buckets []Bucket, err error)

func (*Meta) GetClusters

func (m *Meta) GetClusters() (cluster []Cluster, err error)

func (*Meta) GetFreezer

func (m *Meta) GetFreezer(bucketName string, objectName string, version string) (freezer *types.Freezer, err error)

func (*Meta) GetFreezerStatus

func (m *Meta) GetFreezerStatus(bucketName string, objectName string, version string) (freezer *types.Freezer, err error)

func (*Meta) GetMultipart

func (m *Meta) GetMultipart(bucketName, objectName, uploadId string) (multipart Multipart, err error)

func (*Meta) GetObject

func (m *Meta) GetObject(bucketName, objectName, reqVersion string, willNeed bool) (object *Object, err error)

func (*Meta) GetUsage

func (m *Meta) GetUsage(bucketName string) (int64, error)

func (*Meta) GetUserBuckets

func (m *Meta) GetUserBuckets(userId string, willNeed bool) (buckets []string, err error)

func (*Meta) GetUserInfo

func (m *Meta) GetUserInfo(userId string) ([]string, error)

func (*Meta) MigrateObject

func (m *Meta) MigrateObject(object *Object) error

func (*Meta) NewTrans

func (m *Meta) NewTrans() (tx Tx, err error)

func (*Meta) PutBucketToLifeCycle

func (m *Meta) PutBucketToLifeCycle(bucket Bucket) error

func (*Meta) PutFreezer

func (m *Meta) PutFreezer(freezer *types.Freezer, status common.RestoreStatus) error

func (*Meta) PutFreezerToGarbageCollection

func (m *Meta) PutFreezerToGarbageCollection(freezer *Freezer) error

func (*Meta) PutObject

func (m *Meta) PutObject(reqCtx RequestContext, object *Object, multipart *Multipart, updateUsage bool) (deltaInfo map[common.StorageClass]int64, err error)

func (*Meta) PutObjectPart

func (m *Meta) PutObjectPart(multipart Multipart, part Part) (deltaSize int64, err error)

func (*Meta) PutObjectToGarbageCollection

func (m *Meta) PutObjectToGarbageCollection(object *Object) error

Generate garbage object and send to Kafka

func (*Meta) RemoveBucketFromLifeCycle

func (m *Meta) RemoveBucketFromLifeCycle(bucket Bucket) error

func (*Meta) RemoveHotObject

func (m *Meta) RemoveHotObject(object *Object, tx Tx) error

func (*Meta) RenameObject

func (m *Meta) RenameObject(object *Object, sourceObject string) error

func (*Meta) ReplaceObjectMetas

func (m *Meta) ReplaceObjectMetas(object *Object) error

func (*Meta) ScanLifeCycle

func (m *Meta) ScanLifeCycle(limit int, marker string) (result ScanLifeCycleResult, err error)

func (*Meta) UpdateFreezerDate

func (m *Meta) UpdateFreezerDate(freezer *types.Freezer) error

func (*Meta) UpdateGlacierObject

func (m *Meta) UpdateGlacierObject(reqCtx RequestContext, targetObject, sourceObject *Object) (err error)

func (*Meta) UpdateObjectAcl

func (m *Meta) UpdateObjectAcl(object *Object) error

func (*Meta) UpdateObjectAttrs

func (m *Meta) UpdateObjectAttrs(object *Object) error

type MetaCache

type MetaCache interface {
	Get(table redis.RedisDatabase, key string,
		onCacheMiss func() (interface{}, error),
		unmarshaller func([]byte) (interface{}, error), willNeed bool) (value interface{}, err error)
	Remove(table redis.RedisDatabase, key string)
	GetCacheHitRatio() float64
}

type QosMeta

type QosMeta struct {
	// contains filtered or unexported fields
}

func NewQosMeta

func NewQosMeta(client client.Client) *QosMeta

func (*QosMeta) AllowReadQuery

func (m *QosMeta) AllowReadQuery(bucketName string) (allow bool)

func (*QosMeta) AllowWriteQuery

func (m *QosMeta) AllowWriteQuery(bucketName string) (allow bool)

func (*QosMeta) NewThrottleReader

func (m *QosMeta) NewThrottleReader(bucketName string, reader io.Reader) *ThrottleReader

func (*QosMeta) NewThrottleWriter

func (m *QosMeta) NewThrottleWriter(bucketName string, writer io.Writer) *ThrottleWriter

type ThrottleReader

type ThrottleReader struct {
	// contains filtered or unexported fields
}

func (*ThrottleReader) Close

func (r *ThrottleReader) Close()

func (*ThrottleReader) Read

func (r *ThrottleReader) Read(p []byte) (int, error)

type ThrottleWriter

type ThrottleWriter struct {
	// contains filtered or unexported fields
}

func (*ThrottleWriter) Close

func (w *ThrottleWriter) Close()

func (*ThrottleWriter) Write

func (w *ThrottleWriter) Write(p []byte) (int, error)