core

Documentation

Index

• func New[IN Job, OUT any](ctx context.Context, scraper Scraper[IN, OUT]) *manager[IN, OUT]
• type CoreRequestProcessor
• type DelegatedOperator
• type Delegator
• type Job
• type ManagerJob
  • func (m *ManagerJob[IN]) SetMaxResultsAllowed(maxResultsAllowed uint64)
• type Output
• type Pipeline
• type PipelineManager
  • func NewPipelineManager[J Job, IN any, OUT any, OR Output[J, OUT]]() *PipelineManager[J, IN, OUT, OR]
  • func (p *PipelineManager[J, IN, OUT, OR]) Add(pipeline Pipeline[J, IN, OUT, OR]) *PipelineManager[J, IN, OUT, OR]
• type Request
  • func (r *Request) Body() any
  • func (r *Request) Headers() map[string]string
  • func (r *Request) MetaData() map[string]any
  • func (r *Request) MetaDataKey(key string) (any, bool)
  • func (r *Request) Method() string
  • func (r *Request) Reset()
  • func (r *Request) SetBody(body any) RequestWriter
  • func (r *Request) SetHeaders(headers map[string]string) RequestWriter
  • func (r *Request) SetMetaData(key string, val any) RequestWriter
  • func (r *Request) SetMethod(method string) RequestWriter
  • func (r *Request) SetUrl(url string) RequestWriter
  • func (r *Request) Url() string
• type RequestWriter
• type Response
  • func NewResponse() *Response
  • func (r *Response) Body() []byte
  • func (r *Response) Headers() http.Header
  • func (r *Response) Reset()
  • func (r *Response) SetBody(body []byte) executer.ResponseWriter
  • func (r *Response) SetHeaders(headers http.Header) executer.ResponseWriter
  • func (r *Response) SetStatusCode(statuscode int) executer.ResponseWriter
  • func (r *Response) StatusCode() int
  • func (r *Response) String() string
• type ResponseCallback
• type ResponseReader
• type Scraper
• type ScraperCore
• type ScraperCoreUtility
• type ScraperDelegation
  • func (s *ScraperDelegation[IN, OUT]) ExRequest(ctx context.Context, req *Request, cb ResponseCallback)
  • func (s *ScraperDelegation[IN, OUT]) NewRequest() *Request

Functions

func New

func New[IN Job, OUT any](ctx context.Context, scraper Scraper[IN, OUT]) *manager[IN, OUT]

Types

type CoreRequestProcessor

type CoreRequestProcessor func(context.Context, *Request, ResponseCallback)

type DelegatedOperator

type DelegatedOperator[IN Job, OUT any] struct {
	// contains filtered or unexported fields
}

type Delegator

type Delegator interface {
	ExRequest(ctx context.Context, req *Request, cb ResponseCallback)
	NewRequest() *Request
}

type Job

type Job interface {
	GetId() string
}

type ManagerJob

type ManagerJob[IN Job] struct {
	ScraperJob IN
	// contains filtered or unexported fields
}

func (*ManagerJob[IN]) SetMaxResultsAllowed

func (m *ManagerJob[IN]) SetMaxResultsAllowed(maxResultsAllowed uint64)

type Output

type Output[IN Job, OUT any] interface {
	Records() OUT
	Error() error
	UpdatedJob() IN
}

type Pipeline

type Pipeline[J Job, IN any, OUT any, OR Output[J, OUT]] interface {
	Open(context.Context) error
	Close()
	ProcessItem(IN, OR, metadata.MetaData) (IN, error)
}

type PipelineManager

type PipelineManager[J Job, IN any, OUT any, OR Output[J, OUT]] struct {
	// contains filtered or unexported fields
}

func NewPipelineManager

func NewPipelineManager[J Job, IN any, OUT any, OR Output[J, OUT]]() *PipelineManager[J, IN, OUT, OR]

func (*PipelineManager[J, IN, OUT, OR]) Add

func (p *PipelineManager[J, IN, OUT, OR]) Add(pipeline Pipeline[J, IN, OUT, OR]) *PipelineManager[J, IN, OUT, OR]

type Request

type Request struct {
	// contains filtered or unexported fields
}

func (*Request) Body

func (r *Request) Body() any

func (*Request) Headers

func (r *Request) Headers() map[string]string

func (*Request) MetaData

func (r *Request) MetaData() map[string]any

func (*Request) MetaDataKey

func (r *Request) MetaDataKey(key string) (any, bool)

func (*Request) Method

func (r *Request) Method() string

func (*Request) Reset

func (r *Request) Reset()

func (*Request) SetBody

func (r *Request) SetBody(body any) RequestWriter

func (*Request) SetHeaders

func (r *Request) SetHeaders(headers map[string]string) RequestWriter

func (*Request) SetMetaData

func (r *Request) SetMetaData(key string, val any) RequestWriter

func (*Request) SetMethod

func (r *Request) SetMethod(method string) RequestWriter

func (*Request) SetUrl

func (r *Request) SetUrl(url string) RequestWriter

func (*Request) Url

func (r *Request) Url() string

type RequestWriter

type RequestWriter interface {
	SetUrl(string) RequestWriter
	SetHeaders(map[string]string) RequestWriter
	SetMethod(string) RequestWriter
	SetBody(any) RequestWriter
	SetMetaData(string, any) RequestWriter
}

type Response

type Response struct {
	// contains filtered or unexported fields
}

func NewResponse

func NewResponse() *Response

func (*Response) Body

func (r *Response) Body() []byte

func (*Response) Headers

func (r *Response) Headers() http.Header

func (*Response) Reset

func (r *Response) Reset()

func (*Response) SetBody

func (r *Response) SetBody(body []byte) executer.ResponseWriter

func (*Response) SetHeaders

func (r *Response) SetHeaders(headers http.Header) executer.ResponseWriter

func (*Response) SetStatusCode

func (r *Response) SetStatusCode(statuscode int) executer.ResponseWriter

setters

func (*Response) StatusCode

func (r *Response) StatusCode() int

func (*Response) String

func (r *Response) String() string

type ResponseCallback

type ResponseCallback func(context.Context, ResponseReader)

type ResponseReader

type ResponseReader interface {
	Headers() http.Header
	Body() []byte
	StatusCode() int
}

type Scraper

type Scraper[IN Job, OUT any] interface {
	ScraperCore[IN, OUT]
	ScraperCoreUtility[IN, OUT]
	SetDelegator(Delegator)
}

type ScraperCore

type ScraperCore[IN Job, OUT any] interface {
	Start(context.Context)
	Stop()
	PushJob(IN)
	PullResult() Output[IN, OUT]
	NewJob() IN
}

type ScraperCoreUtility

type ScraperCoreUtility[IN Job, OUT any] interface {
	StartRequest(context.Context, IN)
}

type ScraperDelegation

type ScraperDelegation[IN Job, OUT any] struct {
	// contains filtered or unexported fields
}

func (*ScraperDelegation[IN, OUT]) ExRequest

func (s *ScraperDelegation[IN, OUT]) ExRequest(ctx context.Context, req *Request, cb ResponseCallback)

func (*ScraperDelegation[IN, OUT]) NewRequest

func (s *ScraperDelegation[IN, OUT]) NewRequest() *Request