crawler

Documentation

Index

• func NewEngine(workerCount uint64) *engine.Engine[string]
• type Config
• type Engine
• type HandleFunc
• type Request
  • func NewRequest(key string, kind engine.Kind, taskFunc TaskFunc) *Request
  • func NewUrlKindRequest(url string, kind engine.Kind, handleFunc HandleFunc) *Request
  • func NewUrlRequest(url string, handleFunc HandleFunc) *Request
• type TaskFunc

Functions

func NewEngine

func NewEngine(workerCount uint64) *engine.Engine[string]

Types

type Config

type Config = engine.Config[string]

type Engine

type Engine = engine.Engine[string]

type HandleFunc

type HandleFunc func(ctx context.Context, url string) ([]*Request, error)

type Request

type Request = engine.Task[string]

func NewRequest

func NewRequest(key string, kind engine.Kind, taskFunc TaskFunc) *Request

func NewUrlKindRequest

func NewUrlKindRequest(url string, kind engine.Kind, handleFunc HandleFunc) *Request

func NewUrlRequest

func NewUrlRequest(url string, handleFunc HandleFunc) *Request

type TaskFunc

type TaskFunc = engine.TaskFunc[string]