sqldb

package
v0.1.1-fix-6-http-2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 26, 2023 License: GPL-3.0 Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// JobTableName defines the job context table name
	JobTableName = "job"
	// ObjectTableName defines the object table name
	ObjectTableName = "object"
	// IntegrityMetaTableName defines the integrity meta table name
	IntegrityMetaTableName = "integrity_meta"
	// SpInfoTableName defines the SP info table name
	SpInfoTableName = "sp_info"
	// StorageParamsTableName defines the storage params info table name
	StorageParamsTableName = "storage_params"
	// BucketTrafficTableName defines the bucket traffic table name, which is used for recoding the used quota by bucket
	BucketTrafficTableName = "bucket_traffic"
	// ReadRecordTableName defines the read record table name
	ReadRecordTableName = "read_record"
	// ServiceConfigTableName defines the SP configuration table name
	ServiceConfigTableName = "service_config"
)

define table name constant

Variables

This section is empty.

Functions

func GetCurrentTimestampUs 

func GetCurrentTimestampUs() int64

GetCurrentTimestampUs return a microsecond timestamp

func GetCurrentUnixTime 

func GetCurrentUnixTime() int64

GetCurrentUnixTime return a second timestamp

func GetCurrentYearMonth 

func GetCurrentYearMonth() string

GetCurrentYearMonth get current year and month

func InitDB 

func InitDB(config *config.SQLDBConfig) (*gorm.DB, error)

InitDB init a db instance

func LoadDBConfigFromEnv 

func LoadDBConfigFromEnv(config *config.SQLDBConfig)

LoadDBConfigFromEnv load db user and password from env vars

func OverrideConfigVacancy added in v0.1.1 

func OverrideConfigVacancy(config *config.SQLDBConfig)

OverrideConfigVacancy override the SQLDB param zero value

func TimeToYearMonth 

func TimeToYearMonth(t time.Time) string

TimeToYearMonth convent time.Time to YYYY-MM string

func TimestampSecToTime 

func TimestampSecToTime(timeUnix int64) time.Time

TimestampSecToTime convert a second timestamp to time.Time

func TimestampUsToTime 

func TimestampUsToTime(ts int64) time.Time

TimestampUsToTime convert a microsecond timestamp to time.Time

Types

type Batch 

type Batch interface {
	// Put inserts the given value into the key-value data store.
	Put(key interface{}, value interface{}) error

	// Delete removes the key from the key-value data store.
	Delete(key interface{}) error

	// ValueSize retrieves the amount of data queued up for writing.
	ValueSize() int

	// Write flushes any accumulated data to disk.
	Write() error

	// Reset resets the batch for reuse.
	Reset()
}

Batch is a write-only database that commits changes to its host database when Write is called. A batch cannot be used concurrently.

type Batcher 

type Batcher interface {
	// NewBatch creates a write-only database that buffers changes to its host db
	// until a final write is called.
	NewBatch() Batch

	// NewBatchWithSize creates a write-only database batch with pre-allocated buffer.
	NewBatchWithSize(size int) Batch
}

Batcher wraps the NewBatch method of a backing data store.

type BucketQuota 

type BucketQuota struct {
	ReadQuotaSize uint64
}

BucketQuota defines read quota of a bucket

type BucketTraffic 

type BucketTraffic struct {
	BucketID         uint64
	YearMonth        string // YearMonth is traffic's YearMonth, format "2023-02"
	BucketName       string
	ReadConsumedSize uint64
	ReadQuotaSize    uint64
	ModifyTime       int64
}

BucketTraffic is record traffic by year and month

type BucketTrafficTable 

type BucketTrafficTable struct {
	BucketID uint64 `gorm:"primary_key"`
	Month    string `gorm:"primary_key"`

	BucketName       string
	ReadConsumedSize uint64
	// ReadQuotaSize = the greenfield chain bucket quota + the sp default free quota
	ReadQuotaSize uint64
	ModifiedTime  time.Time
}

BucketTrafficTable table schema

func (BucketTrafficTable) TableName 

func (BucketTrafficTable) TableName() string

TableName is used to set BucketTraffic Schema's table name in database

type IntegrityMeta 

type IntegrityMeta struct {
	ObjectID      uint64
	Checksum      [][]byte
	IntegrityHash []byte
	Signature     []byte
}

IntegrityMeta defines the payload integrity hash and piece checksum with objectID

type IntegrityMetaTable 

type IntegrityMetaTable struct {
	ObjectID      uint64 `gorm:"primary_key"`
	PieceHashList string
	IntegrityHash string
	Signature     string
}

IntegrityMetaTable table schema

func (IntegrityMetaTable) TableName 

func (IntegrityMetaTable) TableName() string

TableName is used to set IntegrityMetaTable schema's table name in database

type Iteratee 

type Iteratee interface {
	// NewIterator creates a binary-alphabetical iterator over a subset
	// of database content with a particular key prefix, starting at a particular
	// initial key (or after, if it does not exist).
	//
	// Note: This method assumes that the prefix is NOT part of the start, so there's
	// no need for the caller to prepend the prefix to the start
	NewIterator(start interface{}) Iterator
}

Iteratee wraps the NewIterator methods of a backing data store.

type Iterator 

type Iterator interface {
	// IsValid return true if current element is valid.
	IsValid() bool

	// Next move to next
	Next()

	// Error returns any accumulated error. Exhausting all the key/value pairs
	// is not considered to be an error.
	Error() error

	// Key returns the key of the current key/value pair, or nil if done. The caller
	// should not modify the contents of the returned slice, and its contents may
	// change on the next call to Next.
	Key() interface{}

	// Value returns the value of the current key/value pair, or nil if done. The
	// caller should not modify the contents of the returned slice, and its contents
	// may change on the next call to Next.
	Value() interface{}

	// Release releases associated resources. Release should always succeed and can
	// be called multiple times without causing error.
	Release()
}

Iterator iterates over a database's key/value pairs in ascending key order.

When it encounters an error any seek will return false and will yield no key/ value pairs. The error can be queried by calling the Error method. Calling Release is still necessary.

An iterator must be released after use, but it is not necessary to read an iterator until exhaustion. An iterator is not safe for concurrent use, but it is safe to use multiple iterators concurrently.

type Job 

type Job interface {
	// CreateUploadJob create upload job and return job context
	CreateUploadJob(objectInfo *storagetypes.ObjectInfo) (*servicetypes.JobContext, error)
	// UpdateJobState update the state of a job by object id
	UpdateJobState(objectID uint64, state servicetypes.JobState) error
	// GetJobByID get job context by job id and return job context
	GetJobByID(jobID uint64) (*servicetypes.JobContext, error)
	// GetJobByObjectID get job context by object id
	GetJobByObjectID(objectID uint64) (*servicetypes.JobContext, error)
}

Job interface which contains job related to object id interface

type JobTable 

type JobTable struct {
	JobID        uint64 `gorm:"primary_key;autoIncrement"`
	JobType      int32
	JobState     int32
	JobErrorCode uint32
	CreatedTime  time.Time
	ModifiedTime time.Time
}

JobTable table schema

func (JobTable) TableName 

func (JobTable) TableName() string

TableName is used to set JobTable Schema's table name in database

type Object 

type Object interface {
	// GetObjectInfo get object info by object id
	GetObjectInfo(objectID uint64) (*storagetypes.ObjectInfo, error)
	// SetObjectInfo set(maybe overwrite) object info by object id
	SetObjectInfo(objectID uint64, objectInfo *storagetypes.ObjectInfo) error
}

Object interface which contains get and set object info interface

type ObjectIntegrity 

type ObjectIntegrity interface {
	// GetObjectIntegrity get integrity meta info by object id
	GetObjectIntegrity(objectID uint64) (*IntegrityMeta, error)
	// SetObjectIntegrity set(maybe overwrite) integrity hash info to db
	SetObjectIntegrity(integrity *IntegrityMeta) error
}

ObjectIntegrity abstract object integrity interface

type ObjectTable 

type ObjectTable struct {
	ObjectID             uint64 `gorm:"primary_key"`
	JobID                uint64 `gorm:"index:job_to_object"` // Job.JobID
	Owner                string
	BucketName           string
	ObjectName           string
	PayloadSize          uint64
	Visibility           int32
	ContentType          string
	CreatedAtHeight      int64
	ObjectStatus         int32
	RedundancyType       int32
	SourceType           int32
	SpIntegrityHash      string
	SecondarySpAddresses string
}

ObjectTable table schema

func (ObjectTable) TableName 

func (ObjectTable) TableName() string

TableName is used to set ObjectTable Schema's table name in database

type ReadRecord 

type ReadRecord struct {
	BucketID        uint64
	ObjectID        uint64
	UserAddress     string
	BucketName      string
	ObjectName      string
	ReadSize        uint64
	ReadTimestampUs int64
}

ReadRecord defines a read request record, will decrease the bucket read quota

type ReadRecordTable 

type ReadRecordTable struct {
	ReadRecordID uint64 `gorm:"primary_key;autoIncrement"`

	BucketID        uint64 `gorm:"index:bucket_to_read_record"`
	ObjectID        uint64 `gorm:"index:object_to_read_record"`
	UserAddress     string `gorm:"index:user_to_read_record"`
	ReadTimestampUs int64  `gorm:"index:time_to_read_record"` // microsecond timestamp

	BucketName string
	ObjectName string
	ReadSize   uint64
}

ReadRecordTable table schema

func (ReadRecordTable) TableName 

func (ReadRecordTable) TableName() string

TableName is used to set ReadRecord Schema's table name in database

type SPDB 

type SPDB interface {
	Job
	Object
	ObjectIntegrity
	Traffic
	SPInfo
	StorageParam
}

SPDB contains all the methods required by sql database

type SPInfo 

type SPInfo interface {
	// UpdateAllSp update all sp info, delete old sp info
	UpdateAllSp(spList []*sptypes.StorageProvider) error
	// FetchAllSp if status is nil return all sp info; otherwise return sp info by status
	FetchAllSp(status ...sptypes.Status) ([]*sptypes.StorageProvider, error)
	// FetchAllSpWithoutOwnSp if status is nil return all sp info without own sp;
	// otherwise return sp info by status without own sp
	FetchAllSpWithoutOwnSp(status ...sptypes.Status) ([]*sptypes.StorageProvider, error)
	// GetSpByAddress return sp info by address and addressType
	GetSpByAddress(address string, addressType SpAddressType) (*sptypes.StorageProvider, error)
	// GetSpByEndpoint return sp info by endpoint
	GetSpByEndpoint(endpoint string) (*sptypes.StorageProvider, error)

	// GetOwnSpInfo return own sp info
	GetOwnSpInfo() (*sptypes.StorageProvider, error)
	// SetOwnSpInfo set(maybe overwrite) own sp info
	SetOwnSpInfo(sp *sptypes.StorageProvider) error
}

SPInfo interface

type ServiceConfig 

type ServiceConfig interface {
	GetAllServiceConfigs() (string, string, error)
	SetAllServiceConfigs(version, config string) error
}

ServiceConfig defines a series of reading and setting service config interfaces

type ServiceConfigTable 

type ServiceConfigTable struct {
	ConfigVersion string `gorm:"primary_key"`
	ServiceConfig string
}

ServiceConfigTable table schema

func (ServiceConfigTable) TableName 

func (ServiceConfigTable) TableName() string

TableName is used to set ServiceConfigTable Schema's table name in database

type SpAddressType 

type SpAddressType int32

SpAddressType identify address type of SP 

const (
	OperatorAddressType SpAddressType = iota + 1
	FundingAddressType
	SealAddressType
	ApprovalAddressType
)

type SpDBImpl 

type SpDBImpl struct {
	// contains filtered or unexported fields
}

SpDBImpl storage provider database, implements SPDB interface

func NewSpDB 

func NewSpDB(config *config.SQLDBConfig) (*SpDBImpl, error)

NewSpDB return a database instance

func (*SpDBImpl) CheckQuotaAndAddReadRecord 

func (s *SpDBImpl) CheckQuotaAndAddReadRecord(record *ReadRecord, quota *BucketQuota) error

CheckQuotaAndAddReadRecord check current quota, and add read record TODO: Traffic statistics may be inaccurate in extreme cases, optimize it in the future

func (*SpDBImpl) CreateUploadJob 

func (s *SpDBImpl) CreateUploadJob(objectInfo *storagetypes.ObjectInfo) (*servicetypes.JobContext, error)

CreateUploadJob create JobTable record and ObjectTable record; use JobID field for association

func (*SpDBImpl) FetchAllSp 

func (s *SpDBImpl) FetchAllSp(status ...sptypes.Status) ([]*sptypes.StorageProvider, error)

FetchAllSp get all sp info

func (*SpDBImpl) FetchAllSpWithoutOwnSp 

func (s *SpDBImpl) FetchAllSpWithoutOwnSp(status ...sptypes.Status) ([]*sptypes.StorageProvider, error)

FetchAllSpWithoutOwnSp get all spp info without own sp info, own sp is identified by is_own field in db

func (*SpDBImpl) GetAllServiceConfigs 

func (s *SpDBImpl) GetAllServiceConfigs() (string, string, error)

GetAllServiceConfigs query service config table to get all service configs

func (*SpDBImpl) GetBucketReadRecord 

func (s *SpDBImpl) GetBucketReadRecord(bucketID uint64, timeRange *TrafficTimeRange) ([]*ReadRecord, error)

GetBucketReadRecord return bucket record list by time range

func (*SpDBImpl) GetBucketTraffic 

func (s *SpDBImpl) GetBucketTraffic(bucketID uint64, yearMonth string) (*BucketTraffic, error)

GetBucketTraffic return bucket traffic info

func (*SpDBImpl) GetJobByID 

func (s *SpDBImpl) GetJobByID(jobID uint64) (*servicetypes.JobContext, error)

GetJobByID query JobTable by jobID and convert to service/types.JobContext

func (*SpDBImpl) GetJobByObjectID 

func (s *SpDBImpl) GetJobByObjectID(objectID uint64) (*servicetypes.JobContext, error)

GetJobByObjectID query JobTable by jobID and convert to service/types.JobContext

func (*SpDBImpl) GetObjectInfo 

func (s *SpDBImpl) GetObjectInfo(objectID uint64) (*storagetypes.ObjectInfo, error)

GetObjectInfo query ObjectTable by objectID and convert to storage/types.ObjectInfo.

func (*SpDBImpl) GetObjectIntegrity 

func (s *SpDBImpl) GetObjectIntegrity(objectID uint64) (*IntegrityMeta, error)

GetObjectIntegrity return the integrity hash info

func (*SpDBImpl) GetObjectReadRecord 

func (s *SpDBImpl) GetObjectReadRecord(objectID uint64, timeRange *TrafficTimeRange) ([]*ReadRecord, error)

GetObjectReadRecord return object record list by time range

func (*SpDBImpl) GetOwnSpInfo 

func (s *SpDBImpl) GetOwnSpInfo() (*sptypes.StorageProvider, error)

GetOwnSpInfo query own sp info in db

func (*SpDBImpl) GetReadRecord 

func (s *SpDBImpl) GetReadRecord(timeRange *TrafficTimeRange) ([]*ReadRecord, error)

GetReadRecord return record list by time range

func (*SpDBImpl) GetSpByAddress 

func (s *SpDBImpl) GetSpByAddress(address string, addressType SpAddressType) (*sptypes.StorageProvider, error)

GetSpByAddress query sp info in db by address and address type

func (*SpDBImpl) GetSpByEndpoint 

func (s *SpDBImpl) GetSpByEndpoint(endpoint string) (*sptypes.StorageProvider, error)

GetSpByEndpoint query sp info by endpoint

func (*SpDBImpl) GetStorageParams 

func (s *SpDBImpl) GetStorageParams() (*storagetypes.Params, error)

GetStorageParams query storage params in db

func (*SpDBImpl) GetUserReadRecord 

func (s *SpDBImpl) GetUserReadRecord(userAddress string, timeRange *TrafficTimeRange) ([]*ReadRecord, error)

GetUserReadRecord return user record list by time range

func (*SpDBImpl) SetAllServiceConfigs 

func (s *SpDBImpl) SetAllServiceConfigs(version, config string) error

SetAllServiceConfigs set service configs to db; if there is no data in db, insert a new record otherwise update data in db

func (*SpDBImpl) SetObjectInfo 

func (s *SpDBImpl) SetObjectInfo(objectID uint64, objectInfo *storagetypes.ObjectInfo) error

SetObjectInfo set ObjectTable's record by objectID

func (*SpDBImpl) SetObjectIntegrity 

func (s *SpDBImpl) SetObjectIntegrity(meta *IntegrityMeta) error

SetObjectIntegrity put(overwrite) integrity hash info to db

func (*SpDBImpl) SetOwnSpInfo 

func (s *SpDBImpl) SetOwnSpInfo(sp *sptypes.StorageProvider) error

SetOwnSpInfo set(maybe overwrite) own sp info to db

func (*SpDBImpl) SetStorageParams 

func (s *SpDBImpl) SetStorageParams(params *storagetypes.Params) error

SetStorageParams set(maybe overwrite) storage params to db

func (*SpDBImpl) UpdateAllSp 

func (s *SpDBImpl) UpdateAllSp(spList []*sptypes.StorageProvider) error

UpdateAllSp update(maybe overwrite) all sp info in db

func (*SpDBImpl) UpdateJobState 

func (s *SpDBImpl) UpdateJobState(objectID uint64, state servicetypes.JobState) error

UpdateJobState update JobTable record's state

type SpInfoTable 

type SpInfoTable struct {
	OperatorAddress string `gorm:"primary_key"`
	IsOwn           bool   `gorm:"primary_key"`
	FundingAddress  string
	SealAddress     string
	ApprovalAddress string
	TotalDeposit    string
	Status          int32
	Endpoint        string
	Moniker         string
	Identity        string
	Website         string
	SecurityContact string
	Details         string
}

SpInfoTable table schema

func (SpInfoTable) TableName 

func (SpInfoTable) TableName() string

TableName is used to set SpInfoTable Schema's table name in database

type StorageParam 

type StorageParam interface {
	// GetStorageParams return storage params
	GetStorageParams() (*storagetypes.Params, error)
	// SetStorageParams set(maybe overwrite) storage params
	SetStorageParams(params *storagetypes.Params) error
}

StorageParam interface

type StorageParamsTable 

type StorageParamsTable struct {
	ID                      int64 `gorm:"primary_key;autoIncrement"`
	MaxSegmentSize          uint64
	RedundantDataChunkNum   uint32
	RedundantParityChunkNum uint32
	MaxPayloadSize          uint64
}

StorageParamsTable table schema

func (StorageParamsTable) TableName 

func (StorageParamsTable) TableName() string

TableName is used to set StorageParamsTable Schema's table name in database

type Traffic 

type Traffic interface {
	// CheckQuotaAndAddReadRecord create bucket traffic firstly if bucket is not existed,
	// and check whether the added traffic record exceeds the quota, if it exceeds the quota,
	// it will return error, Otherwise, add a record and return nil.
	CheckQuotaAndAddReadRecord(record *ReadRecord, quota *BucketQuota) error

	// GetBucketTraffic return bucket traffic info,
	// notice maybe return (nil, nil) while there is no bucket traffic
	GetBucketTraffic(bucketID uint64, yearMonth string) (*BucketTraffic, error)

	// GetReadRecord return record list by time range
	GetReadRecord(timeRange *TrafficTimeRange) ([]*ReadRecord, error)

	// GetBucketReadRecord return bucket record list by time range
	GetBucketReadRecord(bucketID uint64, timeRange *TrafficTimeRange) ([]*ReadRecord, error)

	// GetObjectReadRecord return object record list by time range
	GetObjectReadRecord(objectID uint64, timeRange *TrafficTimeRange) ([]*ReadRecord, error)

	// GetUserReadRecord return user record list by time range
	GetUserReadRecord(userAddress string, timeRange *TrafficTimeRange) ([]*ReadRecord, error)
}

Traffic define a series of traffic interfaces

type TrafficTimeRange 

type TrafficTimeRange struct {
	StartTimestampUs int64
	EndTimestampUs   int64
	LimitNum         int // is unlimited if LimitNum <= 0
}

TrafficTimeRange is used by query, return records in [StartTimestampUs, EndTimestampUs)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.