utils

Documentation

Index

• Variables
• func BatchFullOrTimeout(runnerName string, stopped *int32, batchLen, batchSize int64, ...) bool
• func CheckNotExistFile(runnerName string, expireMap map[string]int64)
• func DeepCopyByGob(dst, src interface{})
• func DeepCopyByJSON(dst, src interface{})
• func GetFiles(runnerName, dirPath string) (map[string]string, error)
• func GetParseTestData(line string, size int) []string
• func IsDir(path string) bool
• func IsExist(path string) bool
• func IsGzipped(data []byte) bool
• func MongoDail(host string, mode string, syncTimeoutInS int64) (session *mgo.Session, err error)
• func SetMode(s *mgo.Session, modeFriendly string, refresh bool)
• func UpdateExpireMap(runnerName string, fileMap map[string]string, expireMap map[string]int64)
• func WriteZipToFile(zipf *zip.File, filename string) error
• type Backoff
  • func NewBackoff(factor, threshold float64, min, max time.Duration) *Backoff
  • func (b *Backoff) Attempt() float64
  • func (b *Backoff) Duration() time.Duration
  • func (b *Backoff) ForAttempt(attempt float64) time.Duration
  • func (b *Backoff) Reset()
• type Collection
  • func (c Collection) CloseSession() (err error)
• type MongoConfig
• type Stage
• type Tracker
  • func NewTracker() *Tracker
  • func (t *Tracker) Print() string
  • func (t *Tracker) Reset()
  • func (t *Tracker) Track(stage string)

Variables

var JSONTool = jsoniter.Config{UseNumber: true}.Froze()

Functions

func BatchFullOrTimeout

func BatchFullOrTimeout(runnerName string, stopped *int32, batchLen, batchSize int64, lastSend time.Time,
	maxBatchLen, maxBatchSize, maxBatchInterval int) bool

func CheckNotExistFile

func CheckNotExistFile(runnerName string, expireMap map[string]int64)

func DeepCopyByGob

func DeepCopyByGob(dst, src interface{})

func DeepCopyByJSON

func DeepCopyByJSON(dst, src interface{})

func GetFiles

func GetFiles(runnerName, dirPath string) (map[string]string, error)

获取指定目录下的所有文件和对应inode

func GetParseTestData

func GetParseTestData(line string, size int) []string

获取测试数据

func IsDir

func IsDir(path string) bool

func IsExist

func IsExist(path string) bool

IsExist checks whether a file or directory exists. It returns false when the file or directory does not exist.

func IsGzipped

func IsGzipped(data []byte) bool

gzip 前两位是固定的： https://stackoverflow.com/questions/6059302/how-to-check-if-a-file-is-gzip-compressed

func MongoDail

func MongoDail(host string, mode string, syncTimeoutInS int64) (session *mgo.Session, err error)

------------------------------------------------------------------------

func SetMode

func SetMode(s *mgo.Session, modeFriendly string, refresh bool)

func UpdateExpireMap

func UpdateExpireMap(runnerName string, fileMap map[string]string, expireMap map[string]int64)

func WriteZipToFile

func WriteZipToFile(zipf *zip.File, filename string) error

Types

type Backoff

type Backoff struct {
	// contains filtered or unexported fields
}

func NewBackoff

func NewBackoff(factor, threshold float64, min, max time.Duration) *Backoff

func (*Backoff) Attempt

func (b *Backoff) Attempt() float64

func (*Backoff) Duration

func (b *Backoff) Duration() time.Duration

func (*Backoff) ForAttempt

func (b *Backoff) ForAttempt(attempt float64) time.Duration

func (*Backoff) Reset

func (b *Backoff) Reset()

type Collection

type Collection struct {
	*mgo.Collection
}

func (Collection) CloseSession

func (c Collection) CloseSession() (err error)

type MongoConfig

type MongoConfig struct {
	Host           string `json:"host"`
	DB             string `json:"db"`
	Mode           string `json:"mode"`
	SyncTimeoutInS int64  `json:"timeout"` // 以秒为单位
	Direct         bool   `json:"direct"`
}

type Stage

type Stage struct {
	Duration time.Duration
	Note     string
}

type Tracker

type Tracker struct {
	Start  time.Time
	Stages []Stage
}

func NewTracker

func NewTracker() *Tracker

func (*Tracker) Print

func (t *Tracker) Print() string

func (*Tracker) Reset

func (t *Tracker) Reset()

func (*Tracker) Track

func (t *Tracker) Track(stage string)