entity

package
v0.6.0-beta.20210716.1817 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 16, 2021 License: BSD-3-Clause Imports: 11 Imported by: 3

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Address added in v0.6.0 

type Address struct {
	Host string
	Port string
}

func NewAddress added in v0.6.0 

func NewAddress(opts *AddressOptions) (res *Address)

func NewAddressFromString added in v0.6.0 

func NewAddressFromString(address string) (res *Address, err error)

func (*Address) IsEmpty added in v0.6.0 

func (a *Address) IsEmpty() (res bool)

func (*Address) String added in v0.6.0 

func (a *Address) String() (res string)

func (*Address) Value added in v0.6.0 

func (a *Address) Value() (res interface{})

type AddressOptions added in v0.6.0 

type AddressOptions struct {
	Host string
	Port string
}

type BatchRequestPayload added in v0.6.0 

type BatchRequestPayload struct {
	Ids []primitive.ObjectID `form:"ids" json:"ids"`
}

type BatchRequestPayloadWithStringData added in v0.6.0 

type BatchRequestPayloadWithStringData struct {
	Ids    []primitive.ObjectID `form:"ids" json:"ids"`
	Data   string               `form:"data" json:"data"`
	Fields []string             `form:"fields" json:"fields"`
}

type Color added in v0.6.0 

type Color struct {
	Name string `json:"name"`
	Hex  string `json:"hex"`
}

func (*Color) GetHex added in v0.6.0 

func (c *Color) GetHex() string

func (*Color) GetName added in v0.6.0 

func (c *Color) GetName() string

func (*Color) Value added in v0.6.0 

func (c *Color) Value() interface{}

type Condition added in v0.6.0 

type Condition struct {
	Key   string      `json:"key"`
	Op    string      `json:"op"`
	Value interface{} `json:"value"`
}

type ConfigSpiderData 

type ConfigSpiderData struct {
	// 通用
	Name        string `yaml:"name" json:"name"`
	DisplayName string `yaml:"display_name" json:"display_name"`
	Col         string `yaml:"col" json:"col"`
	Remark      string `yaml:"remark" json:"remark"`
	Type        string `yaml:"type" bson:"type"`

	// 可配置爬虫
	Engine     string            `yaml:"engine" json:"engine"`
	StartUrl   string            `yaml:"start_url" json:"start_url"`
	StartStage string            `yaml:"start_stage" json:"start_stage"`
	Stages     []Stage           `yaml:"stages" json:"stages"`
	Settings   map[string]string `yaml:"settings" json:"settings"`

	// 自定义爬虫
	Cmd string `yaml:"cmd" json:"cmd"`
}

type Dependency 

type Dependency struct {
	Name        string `json:"name"`
	Version     string `json:"version"`
	Description string `json:"description"`
	Installed   bool   `json:"installed"`
}

type DocItem 

type DocItem struct {
	Title    string    `json:"title"`
	Url      string    `json:"url"`
	Path     string    `json:"path"`
	Children []DocItem `json:"children"`
}

type Executable 

type Executable struct {
	Path        string `json:"path"`
	FileName    string `json:"file_name"`
	DisplayName string `json:"display_name"`
}

type Field 

type Field struct {
	Name      string `yaml:"name" json:"name"`
	Css       string `yaml:"css" json:"css"`
	Xpath     string `yaml:"xpath" json:"xpath"`
	Attr      string `yaml:"attr" json:"attr"`
	NextStage string `yaml:"next_stage" json:"next_stage"`
	Remark    string `yaml:"remark" json:"remark"`
}

type FileRequestPayload added in v0.6.0 

type FileRequestPayload struct {
	Path    string `json:"path" form:"path"`
	NewPath string `json:"new_path" form:"new_path"`
	Data    string `json:"data" form:"data"`
}

type Filter added in v0.6.0 

type Filter struct {
	IsOr       bool        `form:"is_or" url:"is_or"`
	Conditions []Condition `json:"conditions"`
}

type FsFileInfo added in v0.6.0 

type FsFileInfo struct {
	Name      string                  `json:"name"`      // file name
	Path      string                  `json:"path"`      // file path
	FullPath  string                  `json:"full_path"` // file full path
	Extension string                  `json:"extension"` // file extension
	Md5       string                  `json:"md5"`       // MD5 hash
	IsDir     bool                    `json:"is_dir"`    // whether it is directory
	FileSize  int64                   `json:"file_size"` // file size (bytes)
	Children  []interfaces.FsFileInfo `json:"children"`  // children for sub-directory
}

func (*FsFileInfo) GetChildren added in v0.6.0 

func (f *FsFileInfo) GetChildren() []interfaces.FsFileInfo

func (*FsFileInfo) GetExtension added in v0.6.0 

func (f *FsFileInfo) GetExtension() string

func (*FsFileInfo) GetFileSize added in v0.6.0 

func (f *FsFileInfo) GetFileSize() int64

func (*FsFileInfo) GetFullPath added in v0.6.0 

func (f *FsFileInfo) GetFullPath() string

func (*FsFileInfo) GetIsDir added in v0.6.0 

func (f *FsFileInfo) GetIsDir() bool

func (*FsFileInfo) GetMd5 added in v0.6.0 

func (f *FsFileInfo) GetMd5() string

func (*FsFileInfo) GetName added in v0.6.0 

func (f *FsFileInfo) GetName() string

func (*FsFileInfo) GetPath added in v0.6.0 

func (f *FsFileInfo) GetPath() string

type GrpcBaseServiceMessage added in v0.6.0 

type GrpcBaseServiceMessage struct {
	ModelId interfaces.ModelId `json:"id"`
	Data    []byte             `json:"d"`
}

func (*GrpcBaseServiceMessage) GetData added in v0.6.0 

func (msg *GrpcBaseServiceMessage) GetData() []byte

func (*GrpcBaseServiceMessage) GetModelId added in v0.6.0 

func (msg *GrpcBaseServiceMessage) GetModelId() interfaces.ModelId

func (*GrpcBaseServiceMessage) ToBytes added in v0.6.0 

func (msg *GrpcBaseServiceMessage) ToBytes() (data []byte)

type GrpcBaseServiceParams added in v0.6.0 

type GrpcBaseServiceParams struct {
	Query       bson.M             `json:"q"`
	Id          primitive.ObjectID `json:"id"`
	Update      bson.M             `json:"u"`
	Doc         interfaces.Model   `json:"d"`
	Fields      []string           `json:"f"`
	FindOptions *mongo.FindOptions `json:"o"`
	Docs        []interface{}      `json:"dl"`
}

func (*GrpcBaseServiceParams) Value added in v0.6.0 

func (params *GrpcBaseServiceParams) Value() interface{}

type GrpcDelegateMessage added in v0.6.0 

type GrpcDelegateMessage struct {
	ModelId interfaces.ModelId             `json:"id"`
	Method  interfaces.ModelDelegateMethod `json:"m"`
	Data    []byte                         `json:"d"`
}

func (*GrpcDelegateMessage) GetData added in v0.6.0 

func (msg *GrpcDelegateMessage) GetData() []byte

func (*GrpcDelegateMessage) GetMethod added in v0.6.0 

func (msg *GrpcDelegateMessage) GetMethod() interfaces.ModelDelegateMethod

func (*GrpcDelegateMessage) GetModelId added in v0.6.0 

func (msg *GrpcDelegateMessage) GetModelId() interfaces.ModelId

func (*GrpcDelegateMessage) ToBytes added in v0.6.0 

func (msg *GrpcDelegateMessage) ToBytes() (data []byte)

type GrpcSubscribe added in v0.6.0 

type GrpcSubscribe struct {
	Stream   grpc.NodeService_SubscribeServer
	Finished chan bool
}

func (*GrpcSubscribe) GetFinished added in v0.6.0 

func (sub *GrpcSubscribe) GetFinished() chan bool

func (*GrpcSubscribe) GetStream added in v0.6.0 

func (sub *GrpcSubscribe) GetStream() grpc.NodeService_SubscribeServer

type Lang 

type Lang struct {
	Name              string   `json:"name"`
	ExecutableName    string   `json:"executable_name"`
	ExecutablePaths   []string `json:"executable_paths"`
	DepExecutablePath string   `json:"dep_executable_path"`
	LockPath          string   `json:"lock_path"`
	InstallScript     string   `json:"install_script"`
	InstallStatus     string   `json:"install_status"`
	DepFileName       string   `json:"dep_file_name"`
	InstallDepArgs    string   `json:"install_dep_cmd"`
	Type              string   `json:"type"`
}

type ListRequestData added in v0.6.0 

type ListRequestData struct {
	PageNum  int    `form:"page_num" json:"page_num"`
	PageSize int    `form:"page_size" json:"page_size"`
	SortKey  string `form:"sort_key" json:"sort_key"`
	Status   string `form:"status" json:"status"`
	Keyword  string `form:"keyword" json:"keyword"`
}

type ListResponse added in v0.6.0 

type ListResponse struct {
	Status  string      `json:"status"`
	Message string      `json:"message"`
	Total   int         `json:"total"`
	Data    interface{} `json:"data"`
	Error   string      `json:"error"`
}

type ModelInfo added in v0.6.0 

type ModelInfo struct {
	Id      interfaces.ModelId
	ColName string
}

type NodeData 

type NodeData struct {
	Key        string `json:"key"`
	Hash       string `json:"hash"`
	Ip         string `json:"ip"`
	Hostname   string `json:"hostname"`
	MacAddress string `json:"mac_address"`
	UUID       string `json:"uuid"`
}

type NodeInfo added in v0.6.0 

type NodeInfo struct {
	Key         string `json:"key"`
	IsMaster    bool   `json:"is_master"`
	Name        string `json:"name"`
	Ip          string `json:"ip"`
	Mac         string `json:"mac"`
	Hostname    string `json:"hostname"`
	Description string `json:"description"`
	AuthKey     string `json:"auth_key"`
	MaxRunners  int    `json:"max_runners"`
}

func (NodeInfo) Value added in v0.6.0 

func (n NodeInfo) Value() interface{}

type NodeMessage 

type NodeMessage struct {
	// 通信类别
	Type string `json:"type"`

	// 任务相关
	TaskId string `json:"task_id"` // 任务ID

	// 节点相关
	NodeId string `json:"node_id"` // 节点ID

	// 日志相关
	LogPath string `json:"log_path"` // 日志路径
	Log     string `json:"log"`      // 日志

	// 系统信息
	SysInfo SystemInfo `json:"sys_info"`

	// 爬虫相关
	SpiderId string `json:"spider_id"` //爬虫ID

	// 语言相关
	Lang Lang `json:"lang"`

	// 错误相关
	Error string `json:"error"`
}

type PackageJson 

type PackageJson struct {
	Dependencies map[string]string `json:"dependencies"`
}

type Page 

type Page struct {
	Skip     int
	Limit    int
	PageNum  int
	PageSize int
}

func (*Page) GetPage 

func (p *Page) GetPage(pageNum string, pageSize string)

type Pagination added in v0.6.0 

type Pagination struct {
	Page int `form:"page" url:"page"`
	Size int `form:"size" url:"size"`
}

type Release 

type Release struct {
	Name        string `json:"name"`
	Draft       bool   `json:"draft"`
	PreRelease  bool   `json:"pre_release"`
	PublishedAt string `json:"published_at"`
	Body        string `json:"body"`
}

type ReleaseSlices 

type ReleaseSlices []Release

func (ReleaseSlices) Len 

func (r ReleaseSlices) Len() int

func (ReleaseSlices) Less 

func (r ReleaseSlices) Less(i, j int) bool

func (ReleaseSlices) Swap 

func (r ReleaseSlices) Swap(i, j int)

type Response added in v0.6.0 

type Response struct {
	Status  string      `json:"status"`
	Message string      `json:"message"`
	Data    interface{} `json:"data"`
	Error   string      `json:"error"`
}

type Result added in v0.6.0 

type Result = bson.M

type RpcMessage 

type RpcMessage struct {
	Id      string            `json:"id"`      // 消息ID
	Method  string            `json:"method"`  // 消息方法
	NodeId  string            `json:"node_id"` // 节点ID
	Params  map[string]string `json:"params"`  // 参数
	Timeout int               `json:"timeout"` // 超时
	Result  string            `json:"result"`  // 结果
	Error   string            `json:"error"`   // 错误
}

type ScrapyItem 

type ScrapyItem struct {
	Name   string   `json:"name"`
	Fields []string `json:"fields"`
}

type ScrapySettingParam 

type ScrapySettingParam struct {
	Key   string      `json:"key"`
	Value interface{} `json:"value"`
	Type  string      `json:"type"`
}

type Sort added in v0.6.0 

type Sort struct {
	Key       string `json:"key"`
	Direction string `json:"d"`
}

type SpiderType 

type SpiderType struct {
	Type  string `json:"type" bson:"_id"`
	Count int    `json:"count" bson:"count"`
}

type Stage 

type Stage struct {
	Name      string  `yaml:"name" json:"name"`
	IsList    bool    `yaml:"is_list" json:"is_list"`
	ListCss   string  `yaml:"list_css" json:"list_css"`
	ListXpath string  `yaml:"list_xpath" json:"list_xpath"`
	PageCss   string  `yaml:"page_css" json:"page_css"`
	PageXpath string  `yaml:"page_xpath" json:"page_xpath"`
	PageAttr  string  `yaml:"page_attr" json:"page_attr"`
	Fields    []Field `yaml:"fields" json:"fields"`
}

type StatsDailyItem added in v0.6.0 

type StatsDailyItem struct {
	Date    string `json:"date" bson:"_id"`
	Tasks   int64  `json:"tasks" bson:"tasks"`
	Results int64  `json:"results" bson:"results"`
}

type StatsTasksByStatusItem added in v0.6.0 

type StatsTasksByStatusItem struct {
	Status string `json:"status" bson:"_id"`
	Tasks  int64  `json:"tasks" bson:"tasks"`
}

type StreamMessageTaskData added in v0.6.0 

type StreamMessageTaskData struct {
	TaskId  primitive.ObjectID `json:"task_id"`
	Records []Result           `json:"data"`
	Logs    []string           `json:"logs"`
}

type SystemInfo 

type SystemInfo struct {
	ARCH        string       `json:"arch"`
	OS          string       `json:"os"`
	Hostname    string       `json:"host_name"`
	NumCpu      int          `json:"num_cpu"`
	Executables []Executable `json:"executables"`
}

type TaskMessage added in v0.6.0 

type TaskMessage struct {
	Id    primitive.ObjectID `json:"id"`
	Key   string             `json:"key"`
	Cmd   string             `json:"cmd"`
	Param string             `json:"param"`
}

func (*TaskMessage) ToString added in v0.6.0 

func (m *TaskMessage) ToString() (string, error)

type TaskRunOptions added in v0.6.0 

type TaskRunOptions struct {
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.