meta

README

Meta layer

Storage metadata manipulation helper functions for YIG, including a cache layer for HBase, build on Redis.

Move more methods into this package as more patterns emerge

Documentation

Overview

Copyright 2019 The OpenSDS Authors.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright 2019 The OpenSDS Authors.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright 2019 The OpenSDS Authors.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

• Constants
• Variables
• func AddBucketUsageSyncEvent(bucketName string, usage int64)
• type CacheType
• type Meta
  • func New(cfg MetaConfig) *Meta
  • func (m *Meta) AddGcobjRecord(ctx context.Context, obj *types.Object) error
  • func (m *Meta) DeleteGcobjRecord(ctx context.Context, obj *types.Object) error
  • func (m *Meta) DeleteMultipart(ctx context.Context, multipart Multipart) (err error)
  • func (m *Meta) DeleteObject(ctx context.Context, object *Object) error
  • func (m *Meta) GetBucket(ctx context.Context, bucketName string, willNeed bool) (bucket *Bucket, err error)
  • func (m *Meta) GetMultipart(bucketName, objectName, uploadId string) (Multipart, error)
  • func (m *Meta) GetObject(ctx context.Context, bucketName string, objectName string, versionId string, ...) (object *Object, err error)
  • func (m *Meta) InitBucketUsageCache() error
  • func (m *Meta) ListGcObjs(ctx context.Context, offset, limit int) ([]*types.Object, error)
  • func (m *Meta) MarkObjectAsDeleted(ctx context.Context, object *Object) error
  • func (m *Meta) PutObject(ctx context.Context, object, deleteObj *Object, multipart *Multipart, ...) error
  • func (m *Meta) PutObjectPart(ctx context.Context, multipart Multipart, part Part) (err error)
  • func (m *Meta) Stop()
  • func (m *Meta) UpdateObject4Lifecycle(ctx context.Context, old, new *Object, multipart *Multipart) (err error)
  • func (m *Meta) UpdateObjectMeta(object *Object) error
• type MetaCache
• type MetaConfig
• type SyncEvent
• type SyncWorker

Constants

const (
	BUCKET_CACHE_PREFIX = "bucket:"
	USER_CACHE_PREFIX   = "user:"
)

const (
	SYNC_EVENT_TYPE_UNKNOWN = iota
	SYNC_EVENT_TYPE_BUCKET_USAGE
)

const (
	ENCRYPTION_KEY_LENGTH = 32 // 32 bytes for AES-"256"
)

const (
	MSG_NOT_IMPL = "not implemented."
)

const (
	OBJECT_CACHE_PREFIX = "object:"
)

Variables

var MetaSyncQueue chan SyncEvent

Functions

func AddBucketUsageSyncEvent

func AddBucketUsageSyncEvent(bucketName string, usage int64)

Types

type CacheType

type CacheType int

const (
	NoCache CacheType = iota
	EnableCache
	SimpleCache
)

type Meta

type Meta struct {
	Db     db.DBAdapter
	Logger *log.Logger
	Cache  MetaCache
}

func New

func New(cfg MetaConfig) *Meta

func (*Meta) AddGcobjRecord

func (m *Meta) AddGcobjRecord(ctx context.Context, obj *types.Object) error

func (*Meta) DeleteGcobjRecord

func (m *Meta) DeleteGcobjRecord(ctx context.Context, obj *types.Object) error

func (*Meta) DeleteMultipart

func (m *Meta) DeleteMultipart(ctx context.Context, multipart Multipart) (err error)

func (*Meta) DeleteObject

func (m *Meta) DeleteObject(ctx context.Context, object *Object) error

func (*Meta) GetBucket

func (m *Meta) GetBucket(ctx context.Context, bucketName string, willNeed bool) (bucket *Bucket, err error)

Note the usage info got from this method is possibly not accurate because we don't invalid cache when updating usage. For accurate usage info, use `GetUsage()`

func (*Meta) GetMultipart

func (m *Meta) GetMultipart(bucketName, objectName, uploadId string) (Multipart, error)

func (*Meta) GetObject

func (m *Meta) GetObject(ctx context.Context, bucketName string, objectName string, versionId string, willNeed bool) (object *Object, err error)

Object will be updated to cache while willNeed is true

func (*Meta) InitBucketUsageCache

func (m *Meta) InitBucketUsageCache() error

* init bucket usage cache when meta is newed. *

func (*Meta) ListGcObjs

func (m *Meta) ListGcObjs(ctx context.Context, offset, limit int) ([]*types.Object, error)

func (*Meta) MarkObjectAsDeleted

func (m *Meta) MarkObjectAsDeleted(ctx context.Context, object *Object) error

func (*Meta) PutObject

func (m *Meta) PutObject(ctx context.Context, object, deleteObj *Object, multipart *Multipart, objMap *ObjMap, updateUsage bool) error

func (*Meta) PutObjectPart

func (m *Meta) PutObjectPart(ctx context.Context, multipart Multipart, part Part) (err error)

func (*Meta) Stop

func (m *Meta) Stop()

func (*Meta) UpdateObject4Lifecycle

func (m *Meta) UpdateObject4Lifecycle(ctx context.Context, old, new *Object, multipart *Multipart) (err error)

func (*Meta) UpdateObjectMeta

func (m *Meta) UpdateObjectMeta(object *Object) error

type MetaCache

type MetaCache interface {
	Close()
	Get(table redis.RedisDatabase, prefix, key string,
		onCacheMiss func() (helper.Serializable, error),
		onDeserialize func(map[string]string) (interface{}, error),
		willNeed bool) (value interface{}, err error)
	Remove(table redis.RedisDatabase, prefix, key string)
	GetCacheHitRatio() float64
	Keys(table redis.RedisDatabase, pattern string) ([]string, error)
	HGetInt64(table redis.RedisDatabase, prefix, key, field string) (int64, error)
	HMSet(table redis.RedisDatabase, prefix, key string, fields map[string]interface{}) (string, error)
	HIncrBy(table redis.RedisDatabase, prefix, key, field string, value int64) (int64, error)
}

type MetaConfig

type MetaConfig struct {
	CacheType CacheType
	TidbInfo  string
}

type SyncEvent

type SyncEvent struct {
	Type       int
	RetryTimes int
	Data       interface{}
}

type SyncWorker

type SyncWorker interface {
	Sync(event SyncEvent) error
}