log

README

crawlab-log

Log module for Crawlab

Documentation

Index

• Constants
• Variables
• type Driver
  • func NewLogDriver(logDriverType string, options interface{}) (driver Driver, err error)
  • func NewSeaweedFsLogDriver(options *SeaweedFsLogDriverOptions) (driver Driver, err error)
• type Message
• type Metadata
• type SeaweedFsLogDriver
  • func (d *SeaweedFsLogDriver) Close() (err error)
  • func (d *SeaweedFsLogDriver) Count(pattern string) (count int, err error)
  • func (d *SeaweedFsLogDriver) Find(pattern string, skip, limit int) (lines []string, err error)
  • func (d *SeaweedFsLogDriver) Flush() (err error)
  • func (d *SeaweedFsLogDriver) GetFilePathByPage(page int64) (filePath string)
  • func (d *SeaweedFsLogDriver) GetFilePathsFromSkipAndLimit(skip, limit int) (filePaths []string)
  • func (d *SeaweedFsLogDriver) GetLastFilePath() (filePath string, err error)
  • func (d *SeaweedFsLogDriver) GetLastLogFilePage() (page int64, err error)
  • func (d *SeaweedFsLogDriver) GetLogFiles() (files []goseaweedfs.FilerFileInfo, err error)
  • func (d *SeaweedFsLogDriver) GetMetadata() (metadata Metadata, err error)
  • func (d *SeaweedFsLogDriver) Init() (err error)
  • func (d *SeaweedFsLogDriver) UpdateMetadata() (err error)
  • func (d *SeaweedFsLogDriver) WriteLine(line string) (err error)
  • func (d *SeaweedFsLogDriver) WriteLines(lines []string) (err error)
• type SeaweedFsLogDriverOptions

Constants

const (
	DriverTypeFs    = "fs"    // file system (SeaweedFS)
	DriverTypeMongo = "mongo" // mongodb
	DriverTypeEs    = "es"    // elastic search
)

const (
	MetadataName = "metadata.json"
)

Variables

var (
	ErrInvalidType    = errors.New("invalid type")
	ErrNotImplemented = errors.New("not implemented")
)

Types

type Driver

type Driver interface {
	Init() error
	Close() error
	WriteLine(line string) error
	WriteLines(lines []string) error
	Find(pattern string, skip int, limit int) ([]string, error)
	Count(pattern string) (int, error)
	Flush() error
}

func NewLogDriver

func NewLogDriver(logDriverType string, options interface{}) (driver Driver, err error)

func NewSeaweedFsLogDriver

func NewSeaweedFsLogDriver(options *SeaweedFsLogDriverOptions) (driver Driver, err error)

type Message

type Message struct {
	Id  int64     `json:"id" bson:"id"`
	Msg string    `json:"msg" bson:"msg"`
	Ts  time.Time `json:"ts" bson:"ts"`
}

type Metadata

type Metadata struct {
	Size       int64  `json:"size" bson:"size"`
	TotalLines int64  `json:"total_lines" bson:"total_lines"`
	TotalBytes int64  `json:"total_bytes" bson:"total_bytes"`
	Md5        string `json:"md5" bson:"md5"`
}

type SeaweedFsLogDriver

type SeaweedFsLogDriver struct {
	// contains filtered or unexported fields
}

SeaweedFsLogDriver log driver for seaweedfs logs will be saved as chunks of files log chunk file remote path example: /<baseDir>/<prefix>/00000001

func (*SeaweedFsLogDriver) Close

func (d *SeaweedFsLogDriver) Close() (err error)

func (*SeaweedFsLogDriver) Count

func (d *SeaweedFsLogDriver) Count(pattern string) (count int, err error)

func (*SeaweedFsLogDriver) Find

func (d *SeaweedFsLogDriver) Find(pattern string, skip, limit int) (lines []string, err error)

func (*SeaweedFsLogDriver) Flush

func (d *SeaweedFsLogDriver) Flush() (err error)

func (*SeaweedFsLogDriver) GetFilePathByPage

func (d *SeaweedFsLogDriver) GetFilePathByPage(page int64) (filePath string)

func (*SeaweedFsLogDriver) GetFilePathsFromSkipAndLimit

func (d *SeaweedFsLogDriver) GetFilePathsFromSkipAndLimit(skip, limit int) (filePaths []string)

func (*SeaweedFsLogDriver) GetLastFilePath

func (d *SeaweedFsLogDriver) GetLastFilePath() (filePath string, err error)

func (*SeaweedFsLogDriver) GetLastLogFilePage

func (d *SeaweedFsLogDriver) GetLastLogFilePage() (page int64, err error)

func (*SeaweedFsLogDriver) GetLogFiles

func (d *SeaweedFsLogDriver) GetLogFiles() (files []goseaweedfs.FilerFileInfo, err error)

func (*SeaweedFsLogDriver) GetMetadata

func (d *SeaweedFsLogDriver) GetMetadata() (metadata Metadata, err error)

func (*SeaweedFsLogDriver) Init

func (d *SeaweedFsLogDriver) Init() (err error)

func (*SeaweedFsLogDriver) UpdateMetadata

func (d *SeaweedFsLogDriver) UpdateMetadata() (err error)

func (*SeaweedFsLogDriver) WriteLine

func (d *SeaweedFsLogDriver) WriteLine(line string) (err error)

func (*SeaweedFsLogDriver) WriteLines

func (d *SeaweedFsLogDriver) WriteLines(lines []string) (err error)

type SeaweedFsLogDriverOptions

type SeaweedFsLogDriverOptions struct {
	BaseDir          string // base directory path for log files, default: "logs"
	Prefix           string // directory prefix, default: "test"
	Size             int64  // number of lines per log chunk file, default: 1000
	Padding          int64  // log file name padding, default: 8
	FlushWaitSeconds int64  // wait time to flush buffer, default: 3
	MetadataName     string // metadata file name, set to "metadata.json"
}