common

package
v0.6.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 27, 2019 License: MIT Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ConfCheckSnapTimeout      = "check_snap_timeout"
	ConfCheckRaftTimeout      = "check_raft_timeout"
	ConfIgnoreStartupNoBackup = "ignore_startup_nobackup"
	ConfIgnoreRemoteFileSync  = "ignore_remote_file_sync"
	ConfMaxRemoteRecover      = "max_remote_recover"
)
View Source 
const (
	LOG_ERR int32 = iota
	LOG_WARN
	LOG_INFO
	LOG_DEBUG
	LOG_DETAIL
)
View Source 
const (
	DIR_PERM  = 0755
	FILE_PERM = 0644
)
View Source 
const (
	LearnerRoleLogSyncer = "role_log_syncer"
	LearnerRoleSearcher  = "role_searcher"
)
View Source 
const (
	KVName   = "KV"
	ListName = "LIST"
	HashName = "HASH"
	SetName  = "SET"
	ZSetName = "ZSET"
)
View Source 
const (
	RangeClose uint8 = 0x00
	RangeLOpen uint8 = 0x01
	RangeROpen uint8 = 0x10
	RangeOpen  uint8 = 0x11
)
View Source 
const (
	MAX_PARTITION_NUM = 1024
	MAX_REPLICATOR    = 5
)
View Source 
const (
	// api used by data node
	APIAddNode        = "/cluster/node/add"
	APIAddLearnerNode = "/cluster/node/addlearner"
	APIRemoveNode     = "/cluster/node/remove"
	APIGetMembers     = "/cluster/members"
	APIGetLeader      = "/cluster/leader"
	APICheckBackup    = "/cluster/checkbackup"
	APIGetIndexes     = "/schema/indexes"
	APINodeAllReady   = "/node/allready"
	// check if the namespace raft node is synced and can be elected as leader immediately
	APIIsRaftSynced = "/cluster/israftsynced"
	APITableStats   = "/tablestats"

	// below api for pd
	APIGetSnapshotSyncInfo = "/pd/snapshot_sync_info"
)
View Source 
const (
	DefaultExpirationPolicy = "local_deletion"
)
View Source 
const (
	InternalPrefix = "##"
)
View Source 
const (
	KEYSEP = byte(':')
)
View Source 
const (
	NamespaceTableSeperator = byte(':')
)
View Source 
const (
	SnapWaitTimeout = time.Minute * 20
)

Variables

View Source 
var (
	VerBinary = "unset"
	BuildTime = "unset"
	Commit    = "unset"
)
View Source 
var (
	SCAN_CURSOR_SEP = []byte(";")
	SCAN_NODE_SEP   = []byte(":")
)
View Source 
var (
	ErrInvalidCommand    = errors.New("invalid command")
	ErrStopped           = errors.New("the node stopped")
	ErrQueueTimeout      = errors.New("queue request timeout")
	ErrInvalidArgs       = errors.New("invalid arguments")
	ErrInvalidRedisKey   = errors.New("invalid redis key")
	ErrInvalidScanType   = errors.New("invalid scan type")
	ErrEpochMismatch     = errors.New("epoch mismatch")
	ErrInvalidTableName  = errors.New("table name is invalid")
	ErrInvalidScanCursor = errors.New("invalid scan cursor")
	ErrScanCursorNoTable = errors.New("scan cursor must has table")
	ErrUnexpectError     = errors.New("unexpected error")
	ErrInvalidPrefix     = errors.New("invalid prefix")
	ErrNotSupport        = errors.New("not supported")
)
View Source 
var (
	DefaultSnapCount   = 400000
	DefaultSnapCatchup = 300000
)
View Source 
var (
	MAX_BATCH_NUM         = 5000
	MinScore      float64 = math.Inf(-1)
	MaxScore      float64 = math.Inf(1)
	InvalidScore  int64   = -1 << 63
	MAX_SCAN_JOB          = 10
)
View Source 
var (
	ErrDeprecatedAPI = errors.New("deprecated api")
)
View Source 
var ErrRsyncFailed = errors.New("transfer snapshot failed due to rsync error")
View Source 
var ErrTransferOutofdate = errors.New("waiting transfer snapshot too long, maybe out of date")

Functions

func APIRequest 

func APIRequest(method string, endpoint string, body io.Reader, timeout time.Duration, ret interface{}) (int, error)

stores the result in the value pointed to by ret(must be a pointer)

func Decorate 

func Decorate(f APIHandler, ds ...Decorator) httprouter.Handle

func DeepCopyCmd 

func DeepCopyCmd(cmd redcon.Command) redcon.Command

func DumpDynamicConf added in v0.6.2 

func DumpDynamicConf() []string

func ExtractNamesapce 

func ExtractNamesapce(rawKey []byte) (string, []byte, error)

func ExtractTable 

func ExtractTable(rawKey []byte) ([]byte, []byte, error)

func GetIPv4ForInterfaceName 

func GetIPv4ForInterfaceName(ifname string) string

func GetIntDynamicConf added in v0.6.2 

func GetIntDynamicConf(k string) int

func GetNamespaceAndPartition 

func GetNamespaceAndPartition(fullNamespace string) (string, int)

func GetNsDesp 

func GetNsDesp(ns string, part int) string

func GetStrDynamicConf added in v0.6.2 

func GetStrDynamicConf(k string) string

func IsConfSetted added in v0.6.2 

func IsConfSetted(k string) bool

func IsFullScanCommand 

func IsFullScanCommand(cmd string) bool

func IsInternalTableName 

func IsInternalTableName(tb string) bool

func IsMergeCommand 

func IsMergeCommand(cmd string) bool

func IsMergeIndexSearchCommand 

func IsMergeIndexSearchCommand(cmd string) bool

func IsMergeKeysCommand 

func IsMergeKeysCommand(cmd string) bool

func IsMergeScanCommand 

func IsMergeScanCommand(cmd string) bool

func IsRoleLogSyncer added in v0.6.2 

func IsRoleLogSyncer(role string) bool

func IsValidNamespaceName 

func IsValidNamespaceName(ns string) bool

func NewDeadlineTransport 

func NewDeadlineTransport(timeout time.Duration) *http.Transport

func NewDefaultLogger 

func NewDefaultLogger(module string) *defaultLogger

func RespondV1 

func RespondV1(w http.ResponseWriter, code int, data interface{})

func RunFileSync 

func RunFileSync(remote string, srcPath string, dstPath string, stopCh chan struct{}) error

func SetIntDynamicConf added in v0.6.2 

func SetIntDynamicConf(k string, newV int)

func SetRsyncLimit added in v0.4.3 

func SetRsyncLimit(limit int64)

func SetStrDynamicConf added in v0.6.2 

func SetStrDynamicConf(k string, newV string)

func VerString 

func VerString(app string) string

Types

type APIHandler 

type APIHandler func(http.ResponseWriter, *http.Request, httprouter.Params) (interface{}, error)

func PlainText 

func PlainText(f APIHandler) APIHandler

func V1 

func V1(f APIHandler) APIHandler

type CmdRouter 

type CmdRouter struct {
	// contains filtered or unexported fields
}

func NewCmdRouter 

func NewCmdRouter() *CmdRouter

func (*CmdRouter) GetCmdHandler 

func (r *CmdRouter) GetCmdHandler(name string) (CommandFunc, bool, bool)

func (*CmdRouter) GetMergeCmdHandler 

func (r *CmdRouter) GetMergeCmdHandler(name string) (MergeCommandFunc, bool, bool)

return handler, iswrite, isexist

func (*CmdRouter) Register 

func (r *CmdRouter) Register(isWrite bool, name string, f CommandFunc) bool

func (*CmdRouter) RegisterMerge 

func (r *CmdRouter) RegisterMerge(name string, f MergeCommandFunc) bool

func (*CmdRouter) RegisterWriteMerge 

func (r *CmdRouter) RegisterWriteMerge(name string, f MergeCommandFunc) bool

type CommandFunc 

type CommandFunc func(redcon.Conn, redcon.Command)

type CommandRspFunc 

type CommandRspFunc func(redcon.Conn, redcon.Command, interface{})

type DataType 

type DataType byte

for out use 

const (
	NONE DataType = iota
	KV
	LIST
	HASH
	SET
	ZSET
	ALL
)

func (DataType) String 

func (d DataType) String() string

type Decorator 

type Decorator func(APIHandler) APIHandler

func HttpLog 

func HttpLog(log *LevelLogger, level int32) Decorator

type ExpirationPolicy 

type ExpirationPolicy byte
const (
	// LocalDeletion indicates the expired data would be deleted by the underlying storage system automatically and the logical layer
	// do not need to care about the data expiration. Every node in the cluster should start the 'TTLChecker' of the storage system
	// with this policy.
	LocalDeletion ExpirationPolicy = iota

	// ConsistencyDeletion indicates all the expired data should be deleted through Raft, the underlying storage system should
	// not delete any data and all the expired keys should be sent to the expired channel. Only the leader should starts
	// the 'TTLChecker' with this policy.
	ConsistencyDeletion

	//
	PeriodicalRotation

	UnknownPolicy
)

func StringToExpirationPolicy 

func StringToExpirationPolicy(s string) (ExpirationPolicy, error)

type ExpiredDataBuffer 

type ExpiredDataBuffer interface {
	Write(DataType, []byte) error
}

type FieldPair 

type FieldPair struct {
	Field []byte
	Value []byte
}

type FullScanResult 

type FullScanResult struct {
	Results    []interface{}
	Type       DataType
	NextCursor []byte
	PartionId  string
	Error      error
}

type GLogger 

type GLogger struct {
}

func (*GLogger) Output 

func (gl *GLogger) Output(maxdepth int, s string) error

func (*GLogger) OutputErr 

func (gl *GLogger) OutputErr(maxdepth int, s string) error

func (*GLogger) OutputWarning 

func (gl *GLogger) OutputWarning(maxdepth int, s string) error

type HIndexRespWithValues 

type HIndexRespWithValues struct {
	PKey       []byte
	IndexV     interface{}
	HsetValues [][]byte
	// contains filtered or unexported fields
}

type HsetIndexSchema 

type HsetIndexSchema struct {
	Name       string             `json:"name"`
	IndexField string             `json:"index_field"`
	PrefixLen  int32              `json:"prefix_len"`
	Unique     int32              `json:"unique"`
	ValueType  IndexPropertyDType `json:"value_type"`
	State      IndexState         `json:"state"`
}

func (*HsetIndexSchema) IsValidNewSchema 

func (s *HsetIndexSchema) IsValidNewSchema() bool

type HttpErr 

type HttpErr struct {
	Code int
	Text string
}

func (HttpErr) Error 

func (e HttpErr) Error() string

type IClusterInfo 

type IClusterInfo interface {
	GetClusterName() string
	GetSnapshotSyncInfo(fullNS string) ([]SnapshotSyncInfo, error)
	UpdateMeForNamespaceLeader(fullNS string) (bool, error)
}

type IndexPropertyDType 

type IndexPropertyDType int32
const (
	Int64V  IndexPropertyDType = 0
	Int32V  IndexPropertyDType = 1
	StringV IndexPropertyDType = 2
	MaxVT   IndexPropertyDType = 3
)

type IndexSchema 

type IndexSchema struct {
	HsetIndexes []*HsetIndexSchema `json:"hset_indexes"`
	JSONIndexes []*JSONIndexSchema `json:"json_indexes"`
}

type IndexState 

type IndexState int32
const (
	InitIndex      IndexState = 0
	BuildingIndex  IndexState = 1
	BuildDoneIndex IndexState = 2
	ReadyIndex     IndexState = 3
	DeletedIndex   IndexState = 4
	MaxIndexState  IndexState = 5
)

type InternalCommandFunc 

type InternalCommandFunc func(redcon.Command, int64) (interface{}, error)

type JSONIndexSchema 

type JSONIndexSchema struct {
	State IndexState `json:"state"`
}

type KFVals 

type KFVals struct {
	PK   []byte
	Vals []KVRecordRet
}

type KVRecord 

type KVRecord struct {
	Key   []byte
	Value []byte
}

type KVRecordRet 

type KVRecordRet struct {
	Rec KVRecord
	Err error
}

type KVals 

type KVals struct {
	PK   []byte
	Vals [][]byte
}

type LevelLogger 

type LevelLogger struct {
	Logger Logger
	// contains filtered or unexported fields
}

func NewLevelLogger 

func NewLevelLogger(level int32, l Logger) *LevelLogger

func (*LevelLogger) Debug 

func (ll *LevelLogger) Debug(args ...interface{})

func (*LevelLogger) DebugDepth 

func (ll *LevelLogger) DebugDepth(d int, l string)

func (*LevelLogger) Debugf 

func (ll *LevelLogger) Debugf(f string, args ...interface{})

func (*LevelLogger) Error 

func (ll *LevelLogger) Error(args ...interface{})

func (*LevelLogger) ErrorDepth 

func (ll *LevelLogger) ErrorDepth(d int, l string)

func (*LevelLogger) Errorf 

func (ll *LevelLogger) Errorf(f string, args ...interface{})

func (*LevelLogger) Fatal 

func (ll *LevelLogger) Fatal(args ...interface{})

func (*LevelLogger) Fatalf 

func (ll *LevelLogger) Fatalf(f string, args ...interface{})

func (*LevelLogger) Info 

func (ll *LevelLogger) Info(args ...interface{})

func (*LevelLogger) InfoDepth 

func (ll *LevelLogger) InfoDepth(d int, l string)

func (*LevelLogger) Infof 

func (ll *LevelLogger) Infof(f string, args ...interface{})

func (*LevelLogger) Level 

func (ll *LevelLogger) Level() int32

func (*LevelLogger) Panic 

func (ll *LevelLogger) Panic(args ...interface{})

func (*LevelLogger) Panicf 

func (ll *LevelLogger) Panicf(f string, args ...interface{})

func (*LevelLogger) SetLevel 

func (ll *LevelLogger) SetLevel(l int32)

func (*LevelLogger) Warning 

func (ll *LevelLogger) Warning(args ...interface{})

func (*LevelLogger) Warningf 

func (ll *LevelLogger) Warningf(f string, args ...interface{})

type LogSyncStats 

type LogSyncStats struct {
	Name      string `json:"name"`
	Term      uint64 `json:"term"`
	Index     uint64 `json:"index"`
	Timestamp int64  `json:"timestamp"`
	IsLeader  bool   `json:"is_leader"`
}

type Logger 

type Logger interface {
	Output(maxdepth int, s string) error
	OutputErr(maxdepth int, s string) error
	OutputWarning(maxdepth int, s string) error
}

type MemberInfo 

type MemberInfo struct {
	// the replica id
	ID uint64 `json:"id"`
	// the node id replica belong
	NodeID    uint64 `json:"node_id"`
	GroupName string `json:"group_name"`
	// group id the replica belong (different from namespace)
	GroupID  uint64   `json:"group_id"`
	RaftURLs []string `json:"peer_urls"`
}

func (*MemberInfo) IsEqual 

func (self *MemberInfo) IsEqual(other *MemberInfo) bool

type MergeCommandFunc 

type MergeCommandFunc func(redcon.Command) (interface{}, error)

type MergeLogger 

type MergeLogger struct {
	*LevelLogger
	// contains filtered or unexported fields
}

MergeLogger supports merge logging, which merges repeated log lines and prints summary log lines instead.

For merge logging, MergeLogger prints out the line when the line appears at the first time. MergeLogger holds the same log line printed within defaultMergePeriod, and prints out summary log line at the end of defaultMergePeriod. It stops merging when the line doesn't appear within the defaultMergePeriod.

func NewMergeLogger 

func NewMergeLogger(logger *LevelLogger) *MergeLogger

func (*MergeLogger) MergeError 

func (l *MergeLogger) MergeError(entries ...interface{})

func (*MergeLogger) MergeErrorf 

func (l *MergeLogger) MergeErrorf(format string, args ...interface{})

func (*MergeLogger) MergeInfo 

func (l *MergeLogger) MergeInfo(entries ...interface{})

func (*MergeLogger) MergeInfof 

func (l *MergeLogger) MergeInfof(format string, args ...interface{})

func (*MergeLogger) MergeWarning 

func (l *MergeLogger) MergeWarning(entries ...interface{})

func (*MergeLogger) MergeWarningf 

func (l *MergeLogger) MergeWarningf(format string, args ...interface{})

type MergeWriteCommandFunc 

type MergeWriteCommandFunc func(redcon.Command, interface{}) (interface{}, error)

type NamespaceStats 

type NamespaceStats struct {
	Name              string                 `json:"name"`
	TStats            []TableStats           `json:"table_stats"`
	DBWriteStats      *WriteStats            `json:"db_write_stats"`
	ClusterWriteStats *WriteStats            `json:"cluster_write_stats"`
	InternalStats     map[string]interface{} `json:"internal_stats"`
	EngType           string                 `json:"eng_type"`
	IsLeader          bool                   `json:"is_leader"`
}

type SMCmdRouter 

type SMCmdRouter struct {
	// contains filtered or unexported fields
}

func NewSMCmdRouter 

func NewSMCmdRouter() *SMCmdRouter

func (*SMCmdRouter) GetInternalCmdHandler 

func (r *SMCmdRouter) GetInternalCmdHandler(name string) (InternalCommandFunc, bool)

func (*SMCmdRouter) RegisterInternal 

func (r *SMCmdRouter) RegisterInternal(name string, f InternalCommandFunc) bool

type ScanResult 

type ScanResult struct {
	Keys       [][]byte
	NextCursor []byte
	PartionId  string
	Error      error
}

type ScanStats 

type ScanStats struct {
	ScanCount uint64 `json:"scan_count"`
	// <1024us, 2ms, 4ms, 8ms, 16ms, 32ms, 64ms, 128ms, 256ms, 512ms, 1024ms, 2048ms, 4s, 8s
	ScanLatencyStats [16]int64 `json:"scan_latency_stats"`
}

func (*ScanStats) Copy 

func (ss *ScanStats) Copy() *ScanStats

func (*ScanStats) IncScanCount 

func (ss *ScanStats) IncScanCount()

func (*ScanStats) UpdateLatencyStats 

func (ss *ScanStats) UpdateLatencyStats(latencyUs int64)

func (*ScanStats) UpdateScanStats 

func (ss *ScanStats) UpdateScanStats(latencyUs int64)

type ScorePair 

type ScorePair struct {
	Score  float64
	Member []byte
}

type SearchResultHeap 

type SearchResultHeap []*HIndexRespWithValues

func (SearchResultHeap) Len 

func (sh SearchResultHeap) Len() int

func (SearchResultHeap) Less 

func (sh SearchResultHeap) Less(i, j int) bool

func (*SearchResultHeap) Pop 

func (sh *SearchResultHeap) Pop() interface{}

func (*SearchResultHeap) Push 

func (sh *SearchResultHeap) Push(x interface{})

func (SearchResultHeap) Swap 

func (sh SearchResultHeap) Swap(i, j int)

type ServerStats 

type ServerStats struct {
	// database stats
	NSStats []NamespaceStats `json:"ns_stats"`
	//scan统计
	ScanStats *ScanStats `json:"scan_stats"`
}

type SnapshotSyncInfo 

type SnapshotSyncInfo struct {
	ReplicaID   uint64
	NodeID      uint64
	RemoteAddr  string
	HttpAPIPort string
	DataRoot    string
	RsyncModule string
}

type StoppableListener 

type StoppableListener struct {
	*net.TCPListener
	// contains filtered or unexported fields
}

stoppableListener sets TCP keep-alive timeouts on accepted connections and waits on stopc message

func NewStoppableListener 

func NewStoppableListener(addr string, stopc <-chan struct{}) (*StoppableListener, error)

func (StoppableListener) Accept 

func (ln StoppableListener) Accept() (c net.Conn, err error)

type StringArray 

type StringArray []string

func (*StringArray) Set 

func (a *StringArray) Set(s string) error

func (*StringArray) String 

func (a *StringArray) String() string

type TableStats 

type TableStats struct {
	Name              string `json:"name"`
	KeyNum            int64  `json:"key_num"`
	DiskBytesUsage    int64  `json:"disk_bytes_usage"`
	ApproximateKeyNum int64  `json:"approximate_key_num"`
}

type WriteCmd 

type WriteCmd struct {
	Operation string
	Args      [][]byte
}

type WriteStats 

type WriteStats struct {
	// <100bytes, <1KB, 2KB, 4KB, 8KB, 16KB, 32KB, 64KB, 128KB, 256KB, 512KB, 1MB, 2MB, 4MB
	ValueSizeStats [16]int64 `json:"value_size_stats"`
	// <1024us, 2ms, 4ms, 8ms, 16ms, 32ms, 64ms, 128ms, 256ms, 512ms, 1024ms, 2048ms, 4s, 8s
	WriteLatencyStats [16]int64 `json:"write_latency_stats"`
}

func (*WriteStats) BatchUpdateLatencyStats 

func (ws *WriteStats) BatchUpdateLatencyStats(latencyUs int64, cnt int64)

func (*WriteStats) Copy 

func (ws *WriteStats) Copy() *WriteStats

func (*WriteStats) UpdateLatencyStats 

func (ws *WriteStats) UpdateLatencyStats(latencyUs int64)

func (*WriteStats) UpdateSizeStats 

func (ws *WriteStats) UpdateSizeStats(vSize int64)

func (*WriteStats) UpdateWriteStats 

func (ws *WriteStats) UpdateWriteStats(vSize int64, latencyUs int64)

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.