sdk

README

Crawlab SDK

中文 | English

SDK for Crawlab, including SDK for different programming languages such as Python, Node.js and Java, and a CLI Tool written in Python.

Documentation

Index

• Constants
• func Debug(s string)
• func Debugf(s string, i ...interface{})
• func Error(s string)
• func Errorf(s string, i ...interface{})
• func Fatal(s string)
• func Fatalf(s string, i ...interface{})
• func GetAuthTokenStreamChainInterceptor() grpc.StreamClientInterceptor
• func GetAuthTokenUnaryChainInterceptor() grpc.UnaryClientInterceptor
• func GetClient(opts ...ClientOption) interfaces.Client
• func GetLogger(opts ...LoggerOption) interfaces.Logger
• func GetOssBucket() (bucket *oss.Bucket, err error)
• func GetResultService(opts ...ResultServiceOption) interfaces.ResultService
• func GetTaskId() (id primitive.ObjectID)
• func Info(s string)
• func Infof(s string, i ...interface{})
• func Log(s string)
• func Logf(s string, i ...interface{})
• func OssClientInit() error
• func OssVisitLink(ossPath string, expiredTs int64) (string, error)
• func SaveFileToOss(task entity.OssTask) error
• func SaveItem(items ...entity.Result) error
• func SaveItems(items []entity.Result)
• func Warn(s string)
• func Warnf(s string, i ...interface{})
• type Client
  • func (c *Client) GetModelBaseServiceClient() grpc2.ModelBaseServiceClient
  • func (c *Client) GetModelDelegateClient() grpc2.ModelDelegateClient
  • func (c *Client) GetNodeClient() grpc2.NodeServiceClient
  • func (c *Client) GetPluginClient() grpc2.PluginServiceClient
  • func (c *Client) GetTaskClient() grpc2.TaskServiceClient
• type ClientOption
• type Logger
  • func (l *Logger) Debug(s string)
  • func (l *Logger) Debugf(s string, i ...interface{})
  • func (l *Logger) Error(s string)
  • func (l *Logger) Errorf(s string, i ...interface{})
  • func (l *Logger) Fatal(s string)
  • func (l *Logger) Fatalf(s string, i ...interface{})
  • func (l *Logger) Info(s string)
  • func (l *Logger) Infof(s string, i ...interface{})
  • func (l *Logger) Log(s string)
  • func (l *Logger) Logf(s string, i ...interface{})
  • func (l *Logger) Warn(s string)
  • func (l *Logger) Warnf(s string, i ...interface{})
• type LoggerOption
• type ResultService
  • func (svc *ResultService) SaveItem(items ...entity.Result) error
  • func (svc *ResultService) SaveItems(items []entity.Result)
• type ResultServiceOption

Constants

const (
	TaskIdEnv      = "CRAWLAB_TASK_ID"
	GrpcAddressEnv = "CRAWLAB_GRPC_ADDRESS"
	GrpcAuthKeyEnv = "CRAWLAB_GRPC_AUTH_KEY"
)

const (
	LogLevelDebug = "DEBUG"
	LogLevelInfo  = "INFO"
	LogLevelWarn  = "WARN"
	LogLevelError = "ERROR"
	LogLevelFatal = "FATAL"
)

const GrpcHeaderAuthorization = "authorization"

Functions

func Debug

func Debug(s string)

func Debugf

func Debugf(s string, i ...interface{})

func Error

func Error(s string)

func Errorf

func Errorf(s string, i ...interface{})

func Fatal

func Fatal(s string)

func Fatalf

func Fatalf(s string, i ...interface{})

func GetAuthTokenStreamChainInterceptor

func GetAuthTokenStreamChainInterceptor() grpc.StreamClientInterceptor

func GetAuthTokenUnaryChainInterceptor

func GetAuthTokenUnaryChainInterceptor() grpc.UnaryClientInterceptor

func GetClient

func GetClient(opts ...ClientOption) interfaces.Client

func GetLogger

func GetLogger(opts ...LoggerOption) interfaces.Logger

func GetOssBucket

func GetOssBucket() (bucket *oss.Bucket, err error)

func GetResultService

func GetResultService(opts ...ResultServiceOption) interfaces.ResultService

func GetTaskId

func GetTaskId() (id primitive.ObjectID)

func Info

func Info(s string)

func Infof

func Infof(s string, i ...interface{})

func Log

func Log(s string)

func Logf

func Logf(s string, i ...interface{})

func OssClientInit

func OssClientInit() error

func OssVisitLink

func OssVisitLink(ossPath string, expiredTs int64) (string, error)

func SaveFileToOss

func SaveFileToOss(task entity.OssTask) error

func SaveItem

func SaveItem(items ...entity.Result) error

func SaveItems

func SaveItems(items []entity.Result)

func Warn

func Warn(s string)

func Warnf

func Warnf(s string, i ...interface{})

Types

type Client

type Client struct {

	// dependencies
	ModelDelegateClient    grpc2.ModelDelegateClient
	ModelBaseServiceClient grpc2.ModelBaseServiceClient
	NodeClient             grpc2.NodeServiceClient
	TaskClient             grpc2.TaskServiceClient
	PluginClient           grpc2.PluginServiceClient
	// contains filtered or unexported fields
}

var C *Client

func (*Client) GetModelBaseServiceClient

func (c *Client) GetModelBaseServiceClient() grpc2.ModelBaseServiceClient

func (*Client) GetModelDelegateClient

func (c *Client) GetModelDelegateClient() grpc2.ModelDelegateClient

func (*Client) GetNodeClient

func (c *Client) GetNodeClient() grpc2.NodeServiceClient

func (*Client) GetPluginClient

func (c *Client) GetPluginClient() grpc2.PluginServiceClient

func (*Client) GetTaskClient

func (c *Client) GetTaskClient() grpc2.TaskServiceClient

type ClientOption

type ClientOption func(c interfaces.Client)

type Logger

type Logger struct {
	log.Interface
	// contains filtered or unexported fields
}

var L *Logger

func (*Logger) Debug

func (l *Logger) Debug(s string)

func (*Logger) Debugf

func (l *Logger) Debugf(s string, i ...interface{})

func (*Logger) Error

func (l *Logger) Error(s string)

func (*Logger) Errorf

func (l *Logger) Errorf(s string, i ...interface{})

func (*Logger) Fatal

func (l *Logger) Fatal(s string)

func (*Logger) Fatalf

func (l *Logger) Fatalf(s string, i ...interface{})

func (*Logger) Info

func (l *Logger) Info(s string)

func (*Logger) Infof

func (l *Logger) Infof(s string, i ...interface{})

func (*Logger) Log

func (l *Logger) Log(s string)

func (*Logger) Logf

func (l *Logger) Logf(s string, i ...interface{})

func (*Logger) Warn

func (l *Logger) Warn(s string)

func (*Logger) Warnf

func (l *Logger) Warnf(s string, i ...interface{})

type LoggerOption

type LoggerOption func(l log.Interface)

type ResultService

type ResultService struct {
	// contains filtered or unexported fields
}

var RS *ResultService

func (*ResultService) SaveItem

func (svc *ResultService) SaveItem(items ...entity.Result) error

func (*ResultService) SaveItems

func (svc *ResultService) SaveItems(items []entity.Result)

type ResultServiceOption

type ResultServiceOption func(svc interfaces.ResultService)