core

Documentation

Index

• func New[IN Job, OUT any](ctx context.Context, spider Spider[IN, OUT]) *manager[IN, OUT]
• type CoreRequestProcessor
• type DelegatedOperator
• type Delegator
• type Job
• type Middleware
• type MiddlewareFunc
  • func (mf MiddlewareFunc) RoundTrip(req *http.Request) (*http.Response, error)
• type Output
• type Pipeline
• type PipelineManager
  • func NewPipelineManager[J Job, IN any, OUT any, OR Output[J, OUT]]() *PipelineManager[J, IN, OUT, OR]
  • func (p *PipelineManager[J, IN, OUT, OR]) Add(pipeline Pipeline[J, IN, OUT, OR], err error, required ...bool) *PipelineManager[J, IN, OUT, OR]
• type Request
  • func (r *Request) Body() io.ReadCloser
  • func (r *Request) Headers() map[string]string
  • func (r *Request) MetaData() map[string]any
  • func (r *Request) MetaDataKey(key string) (any, bool)
  • func (r *Request) Method() string
  • func (r *Request) Reset()
  • func (r *Request) SetBody(body any) RequestWriter
  • func (r *Request) SetCookieJar(key string) RequestWriter
  • func (r *Request) SetHeaders(headers map[string]string) RequestWriter
  • func (r *Request) SetMetaData(key string, val any) RequestWriter
  • func (r *Request) SetMethod(method string) RequestWriter
  • func (r *Request) SetUrl(_url string) RequestWriter
  • func (r *Request) Url() *url.URL
• type RequestWriter
• type Response
  • func NewResponse() *Response
  • func (r *Response) Body() io.ReadCloser
  • func (r *Response) Bytes() []byte
  • func (r *Response) Cookies() []*http.Cookie
  • func (r *Response) Headers() http.Header
  • func (r *Response) Reset()
  • func (r *Response) SetBody(body io.ReadCloser) executer.ResponseWriter
  • func (r *Response) SetCookies(cookies []*http.Cookie) executer.ResponseWriter
  • func (r *Response) SetHeaders(headers http.Header) executer.ResponseWriter
  • func (r *Response) SetStatusCode(statuscode int) executer.ResponseWriter
  • func (r *Response) StatusCode() int
• type ResponseCallback
• type ResponseReader
• type Spider
• type SpiderCore
• type SpiderCoreUtility
• type SpiderDelegation
  • func (s *SpiderDelegation[IN, OUT]) ExRequest(ctx context.Context, req *Request, cb ResponseCallback)
  • func (s *SpiderDelegation[IN, OUT]) NewRequest() *Request
  • func (s *SpiderDelegation[IN, OUT]) Yield(output Output[IN, OUT])

Functions

func New

func New[IN Job, OUT any](ctx context.Context, spider Spider[IN, OUT]) *manager[IN, OUT]

Types

type CoreRequestProcessor

type CoreRequestProcessor func(context.Context, *Request, ResponseCallback)

type DelegatedOperator

type DelegatedOperator[IN Job, OUT any] struct {
	// contains filtered or unexported fields
}

type Delegator

type Delegator[IN Job, OUT any] interface {
	ExRequest(ctx context.Context, req *Request, cb ResponseCallback)
	NewRequest() *Request
	Yield(Output[IN, OUT])
}

type Job

type Job interface {
	Id() string
}

type Middleware

type Middleware func(next http.RoundTripper) http.RoundTripper

type MiddlewareFunc

type MiddlewareFunc func(req *http.Request) (*http.Response, error)

func (MiddlewareFunc) RoundTrip

func (mf MiddlewareFunc) RoundTrip(req *http.Request) (*http.Response, error)

type Output

type Output[IN Job, OUT any] interface {
	Records() OUT
	RecordKeys() []string
	RecordsFlat() [][]any
	Error() error
	Job() IN
	IsEmpty() bool
}

type Pipeline

type Pipeline[J Job, IN any, OUT any, OR Output[J, OUT]] interface {
	Open(context.Context) error
	Close()
	ProcessItem(IN, OR, metadata.MetaData) (IN, error)
}

type PipelineManager

type PipelineManager[J Job, IN any, OUT any, OR Output[J, OUT]] struct {
	// contains filtered or unexported fields
}

func NewPipelineManager

func NewPipelineManager[J Job, IN any, OUT any, OR Output[J, OUT]]() *PipelineManager[J, IN, OUT, OR]

func (*PipelineManager[J, IN, OUT, OR]) Add

func (p *PipelineManager[J, IN, OUT, OR]) Add(pipeline Pipeline[J, IN, OUT, OR], err error, required ...bool) *PipelineManager[J, IN, OUT, OR]

type Request

type Request struct {
	// contains filtered or unexported fields
}

func (*Request) Body

func (r *Request) Body() io.ReadCloser

func (*Request) Headers

func (r *Request) Headers() map[string]string

func (*Request) MetaData

func (r *Request) MetaData() map[string]any

func (*Request) MetaDataKey

func (r *Request) MetaDataKey(key string) (any, bool)

func (*Request) Method

func (r *Request) Method() string

func (*Request) Reset

func (r *Request) Reset()

func (*Request) SetBody

func (r *Request) SetBody(body any) RequestWriter

func (*Request) SetCookieJar

func (r *Request) SetCookieJar(key string) RequestWriter

func (*Request) SetHeaders

func (r *Request) SetHeaders(headers map[string]string) RequestWriter

func (*Request) SetMetaData

func (r *Request) SetMetaData(key string, val any) RequestWriter

func (*Request) SetMethod

func (r *Request) SetMethod(method string) RequestWriter

func (*Request) SetUrl

func (r *Request) SetUrl(_url string) RequestWriter

func (*Request) Url

func (r *Request) Url() *url.URL

type RequestWriter

type RequestWriter interface {
	SetUrl(string) RequestWriter
	SetHeaders(map[string]string) RequestWriter
	SetMethod(string) RequestWriter
	SetBody(any) RequestWriter
	SetMetaData(string, any) RequestWriter
	SetCookieJar(string) RequestWriter
}

type Response

type Response struct {
	// contains filtered or unexported fields
}

func NewResponse

func NewResponse() *Response

func (*Response) Body

func (r *Response) Body() io.ReadCloser

func (*Response) Bytes

func (r *Response) Bytes() []byte

func (*Response) Cookies

func (r *Response) Cookies() []*http.Cookie

func (*Response) Headers

func (r *Response) Headers() http.Header

func (*Response) Reset

func (r *Response) Reset()

func (*Response) SetBody

func (r *Response) SetBody(body io.ReadCloser) executer.ResponseWriter

func (*Response) SetCookies

func (r *Response) SetCookies(cookies []*http.Cookie) executer.ResponseWriter

func (*Response) SetHeaders

func (r *Response) SetHeaders(headers http.Header) executer.ResponseWriter

func (*Response) SetStatusCode

func (r *Response) SetStatusCode(statuscode int) executer.ResponseWriter

setters

func (*Response) StatusCode

func (r *Response) StatusCode() int

type ResponseCallback

type ResponseCallback func(context.Context, ResponseReader)

type ResponseReader

type ResponseReader interface {
	Headers() http.Header
	Body() io.ReadCloser
	Bytes() []byte
	StatusCode() int
	Cookies() []*http.Cookie
}

type Spider

type Spider[IN Job, OUT any] interface {
	SpiderCore[IN, OUT]
	SpiderCoreUtility[IN, OUT]
	SetDelegator(Delegator[IN, OUT])
}

type SpiderCore

type SpiderCore[IN Job, OUT any] interface {
	StartRequest(context.Context, IN)
	Close()
	NewJob(string) IN
}

type SpiderCoreUtility

type SpiderCoreUtility[IN Job, OUT any] interface {
	StartRequest(context.Context, IN)
}

type SpiderDelegation

type SpiderDelegation[IN Job, OUT any] struct {
	// contains filtered or unexported fields
}

func (*SpiderDelegation[IN, OUT]) ExRequest

func (s *SpiderDelegation[IN, OUT]) ExRequest(ctx context.Context, req *Request, cb ResponseCallback)

func (*SpiderDelegation[IN, OUT]) NewRequest

func (s *SpiderDelegation[IN, OUT]) NewRequest() *Request

func (*SpiderDelegation[IN, OUT]) Yield

func (s *SpiderDelegation[IN, OUT]) Yield(output Output[IN, OUT])