types

Documentation

Index

• type IPv4Network
  • func (n *IPv4Network) Scan(src interface{}) error
  • func (n IPv4Network) TableName() string
  • func (n IPv4Network) Value() (driver.Value, error)
• type Job
• type Network
  • func (n Network) String() string
• type Processor
• type RequestTpl
  • func (t *RequestTpl) Scan(src interface{}) error
  • func (t RequestTpl) Value() (driver.Value, error)
• type ResponseTpl
  • func (t *ResponseTpl) Scan(src interface{}) error
  • func (t ResponseTpl) Value() (driver.Value, error)
• type Run
• type RunLog
• type RunResult
• type RunStatus
• type Step
• type StepType
• type Steps
• type WorkerJob
• type WorkerRun

Types

type IPv4Network

type IPv4Network struct {
	IP string `json:"ip" gorm:"type:varchar(15);primary_key"`
	Network
	CreatedAt time.Time `json:"createdAt"` // for gorm
}

IPv4Network is a db table: ip_v4_networks

func (*IPv4Network) Scan

func (n *IPv4Network) Scan(src interface{}) error

Scan implements the Scanner interface.

func (IPv4Network) TableName

func (n IPv4Network) TableName() string

TableName is for gorm table name

func (IPv4Network) Value

func (n IPv4Network) Value() (driver.Value, error)

Value implements the driver Valuer interface.

type Job

type Job struct {
	WorkerJob
	Name         string    `json:"name" gorm:"type:varchar(50)"`  // user defined name
	Scope        xtype.Set `json:"scope" gorm:"type:json"`        // required scope, union of the step scopes
	RetryLimit   int       `json:"retryLimit"`                    // tell queen the max retry time for this job run
	NetworkLimit *Network  `json:"networkLimit" gorm:"type:json"` // network limit in run will overwrite it in job define
	QueenSteps   Steps     `json:"queenSteps" gorm:"type:json"`   // job steps runs on queen
	CreatedAt    time.Time `json:"createdAt"`                     // for gorm
	UpdatedAt    time.Time `json:"updatedAt"`                     // for gorm
}

Job struct is the complete definition of job. This is a db table: jobs

type Network

type Network struct {
	Region string `json:"region" gorm:"type:varchar(50)"` // country or region
	State  string `json:"state" gorm:"type:varchar(50)"`  // state or province
	City   string `json:"city" gorm:"type:varchar(50)"`   // city
	ISP    string `json:"isp" gorm:"type:varchar(50)"`    // ISP
}

Network is worker ISP location

func (Network) String

func (n Network) String() string

String log for network

type Processor

type Processor struct {
	ID       string `json:"id"`       // use xid for privacy protection
	Name     string `json:"name"`     // TODO: hide name to worker
	Selector string `json:"selector"` // css selector for html, path for json and xml
	Attr     string `json:"attr"`     // if attr exists,grep the value of it,otherwise grep dom text content
}

Processor can grep data from response content

type RequestTpl

type RequestTpl struct {
	URL       string `json:"url"`                // url template, including fixed query params
	URLVarCnt int    `json:"url_variable_count"` // for job validation

	Method  string       `json:"method"`  // GET/PUT/POST
	Headers *http.Header `json:"headers"` // fixed headers

	Body       string `json:"body"`                // body template
	BodyVarCnt int    `json:"body_variable_count"` // for job validation
}

RequestTpl is Request Job for job

func (*RequestTpl) Scan

func (t *RequestTpl) Scan(src interface{}) error

Scan implements the Scanner interface.

func (RequestTpl) Value

func (t RequestTpl) Value() (driver.Value, error)

Value implements the driver Valuer interface.

type ResponseTpl

type ResponseTpl struct {
	ContentType string      `json:"content_type"` // html/json/xml
	Processors  []Processor `json:"processors"`   // all response process rules
}

ResponseTpl is Response Job for job

func (*ResponseTpl) Scan

func (t *ResponseTpl) Scan(src interface{}) error

Scan implements the Scanner interface.

func (ResponseTpl) Value

func (t ResponseTpl) Value() (driver.Value, error)

Value implements the driver Valuer interface.

type Run

type Run struct {
	WorkerRun
	RunResult
	NetworkLimit *Network   `json:"networkLimit" gorm:"type:json"` // network limit in run will overwrite it in job define
	CreatedAt    time.Time  `json:"createdAt"`                     // queuing
	DispatchedAt *time.Time `json:"dispatchedAt"`                  // queuing -> running
	SucceededAt  *time.Time `json:"succeededAt"`                   // running -> succeeded
	FailedAt     *time.Time `json:"failedAt"`                      // running -> failed
}

Run is the job instance in queen side This is a db table: runs

type RunLog

type RunLog struct {
}

RunLog TODO: we'll save job run log

type RunResult

type RunResult struct {
	Status  RunStatus       `json:"status"`
	Message string          `json:"message"`
	Result  xtype.StringMap `json:"result" gorm:"type:json"`
}

RunResult is the body worker reported to queen.

type RunStatus

type RunStatus string

RunStatus is job instance status in queen side

const (
	RunStatusQueuing   RunStatus = "queuing"
	RunStatusRunning   RunStatus = "running"
	RunStatusSucceeded RunStatus = "succeeded"
	RunStatusFailed    RunStatus = "failed"
)

type Step

type Step struct {
	ID         string   `json:"id" gorm:"type:varchar(20)"`       // user defined id
	Name       string   `json:"name" gorm:"type:varchar(50)"`     // user defined name
	If         string   `json:"if"`                               // You can use the if conditional to prevent a step from running unless a condition is met.
	RetryLimit int      `json:"retryLimit"`                       // tell worker the max retry times for this step
	TimeLimit  int      `json:"timeLimit"`                        // tell worker the max run seconds
	Type       StepType `json:"stepType" gorm:"type:varchar(20)"` // determines content struct
	Content    json.RawMessage
}

Step will be process one by one in a job run.

type StepType

type StepType string

StepType determines the definition of the step

const (
	StepTypeDocker StepType = "docker"
)

All step types

type Steps

type Steps []Step

Steps is a step array, defined for custom db driver

type WorkerJob

type WorkerJob struct {
	ID        string `json:"id" gorm:"type:varchar(20);primary_key"` // use xid for privacy protection
	Version   int    `json:"version"`                                // start from 0, +1 after each update
	Steps     Steps  `json:"steps" gorm:"type:json"`                 // job steps
	TimeLimit int    `json:"timeLimit"`                              // tell worker the max run seconds
}

WorkerJob defines a job which can run on workers.

type WorkerRun

type WorkerRun struct {
	ID         string          `json:"id"`                       // use xid for privacy protection
	JobID      string          `json:"jobID"`                    // job reference
	Job        *WorkerJob      `json:"job"`                      // if worker need
	JobVersion int             `json:"jobVersion"`               // for worker side cache
	Params     xtype.StringMap `json:"params" gorm:"type:json"`  // all params
	Context    xtype.StringMap `json:"context" gorm:"type:json"` // context data in pipeline
}

WorkerRun is the instance of job which will dispatch to worker