crawler

Documentation

Index

• func EndCallback(db *gorm.DB, mod string)
• func NewEngine(workerCount uint) *engine.Engine[string, Prop, Prop]
• type Config
• type Cursor
  • func GetCursor(db *gorm.DB, mod string) (*Cursor, error)
• type Engine
• type HandleFunc
• type Prop
• type Request
  • func EndCallbackReq(db *gorm.DB, mod string) *Request
  • func NewRequest(key string, kind engine.Kind, taskFunc TaskFunc) *Request
  • func NewUrlKindRequest(url string, kind engine.Kind, handleFunc HandleFunc) *Request
  • func NewUrlRequest(url string, handleFunc HandleFunc) *Request
• type TaskFunc
• type TaskMeta
  • func NewTaskMeta(key string) TaskMeta

Functions

func EndCallback

func EndCallback(db *gorm.DB, mod string)

func NewEngine

func NewEngine(workerCount uint) *engine.Engine[string, Prop, Prop]

Types

type Config

type Config = engine.Config[string, Prop, Prop]

type Cursor

type Cursor struct {
	Type   string `json:"type" gorm:"primaryKey"`
	Cursor string
	Prev   string
	Next   string
}

func GetCursor

func GetCursor(db *gorm.DB, mod string) (*Cursor, error)

type Engine

type Engine = engine.Engine[string, Prop, Prop]

type HandleFunc

type HandleFunc func(ctx context.Context, url string) ([]*Request, error)

type Prop

type Prop struct {
}

type Request

type Request = engine.Task[string, Prop]

func EndCallbackReq

func EndCallbackReq(db *gorm.DB, mod string) *Request

func NewRequest

func NewRequest(key string, kind engine.Kind, taskFunc TaskFunc) *Request

func NewUrlKindRequest

func NewUrlKindRequest(url string, kind engine.Kind, handleFunc HandleFunc) *Request

func NewUrlRequest

func NewUrlRequest(url string, handleFunc HandleFunc) *Request

type TaskFunc

type TaskFunc = engine.TaskFunc[string, Prop]

type TaskMeta

type TaskMeta = engine.TaskMeta[string]

func NewTaskMeta

func NewTaskMeta(key string) TaskMeta