remilia

README

Remilia

Remilia is a high-performance web scraping framework designed for efficiency. It enables users to concentrate on extracting and utilizing web content, delegating the complexity of web scraping processes to the framework.

Features

• Clean API & elegant mental model
• Concurrency supporting
• Configurable backoff retry algorithm
• Pre-request & post-response hooks supporting

Example

titleParser := func(in *goquery.Document, put remilia.Put[string]) {
    in.Find("h1").Each(func(i int, s *goquery.Selection) {
        fmt.Println(s.Text())
    })
}

rem, _ := remilia.New()
err := rem.Do(
    rem.Just("https://go.dev/"),
    rem.Unit(titleParser),
)
if err != nil {
    fmt.Println("Error: ", err)
}

Install

go get -u github.com/ShroXd/remilia

License

This project is licensed under the MIT License. See the LICENSE file for details.

Documentation

Index

• func WithWorkLinearAttempt(a uint8) exponentialBackoffOptionFunc
• func WithWorkMaxAttempt(a uint8) exponentialBackoffOptionFunc
• func WithWorkMaxDelay(d time.Duration) exponentialBackoffOptionFunc
• func WithWorkMinDelay(d time.Duration) exponentialBackoffOptionFunc
• func WithWorkMultiplier(m float64) exponentialBackoffOptionFunc
• type Bucket
  • func NewBucket(clock Clock, capacity int64, fillInterval time.Duration, fillQuantum int64, ...) *Bucket
  • func (b *Bucket) Take(count int64) time.Duration
  • func (b *Bucket) Wrap(op func() error) ExecutableFunc
• type Client
• type ClientOptionFunc
  • func WithBaseURL(url string) ClientOptionFunc
  • func WithHeaders(headers map[string]string) ClientOptionFunc
  • func WithLinearAttempt(a uint8) ClientOptionFunc
  • func WithMaxAttempt(a uint8) ClientOptionFunc
  • func WithMaxDelay(d time.Duration) ClientOptionFunc
  • func WithMinDelay(d time.Duration) ClientOptionFunc
  • func WithMultiplier(m float64) ClientOptionFunc
  • func WithPostResponseHooks(hooks ...ResponseHook) ClientOptionFunc
  • func WithPreRequestHooks(hooks ...RequestHook) ClientOptionFunc
  • func WithTimeout(timeout time.Duration) ClientOptionFunc
  • func WithTransformer(transformer transform.Transformer) ClientOptionFunc
  • func WithUserAgentGenerator(fn func() string) ClientOptionFunc
• type Clock
• type ExecutableFunc
• type Get
• type LayerFunc
• type Limiter
• type Logger
• type Put
• type Remilia
  • func New(opts ...RemiliaOptionFunc) (*Remilia, error)
  • func (r *Remilia) AddLayer(fn LayerFunc, opts ...StageOptionFunc) actionLayerDef[*Request]
  • func (r *Remilia) Do(pd providerDef[*Request], stageDefs ...actionLayerDef[*Request]) error
  • func (r *Remilia) URLProvider(urlStr string) providerDef[*Request]
• type RemiliaOptionFunc
  • func WithClientOptions(opts ...ClientOptionFunc) RemiliaOptionFunc
  • func WithLayerOptions(opts ...StageOptionFunc) RemiliaOptionFunc
  • func WithLogger(logger Logger) RemiliaOptionFunc
• type Request
• type RequestHook
• type Response
• type ResponseHook
• type StageOptionFunc
  • func WithConcurrency(concurrency uint) StageOptionFunc
  • func WithInputBufferSize(size uint) StageOptionFunc

Functions

func WithWorkLinearAttempt

func WithWorkLinearAttempt(a uint8) exponentialBackoffOptionFunc

func WithWorkMaxAttempt

func WithWorkMaxAttempt(a uint8) exponentialBackoffOptionFunc

func WithWorkMaxDelay

func WithWorkMaxDelay(d time.Duration) exponentialBackoffOptionFunc

func WithWorkMinDelay

func WithWorkMinDelay(d time.Duration) exponentialBackoffOptionFunc

func WithWorkMultiplier

func WithWorkMultiplier(m float64) exponentialBackoffOptionFunc

Types

type Bucket

type Bucket struct {
	// contains filtered or unexported fields
}

func NewBucket

func NewBucket(clock Clock, capacity int64, fillInterval time.Duration, fillQuantum int64, availableTokens int64) *Bucket

TODO: fuck these params' type, we should use time instead of fucking int64

func (*Bucket) Take

func (b *Bucket) Take(count int64) time.Duration

TODO: use tick instead of time

func (*Bucket) Wrap

func (b *Bucket) Wrap(op func() error) ExecutableFunc

type Client

type Client struct {
	// contains filtered or unexported fields
}

type ClientOptionFunc

type ClientOptionFunc optionFunc[*Client]

TODO: is this a good preactice to mixin otps for network request and custom functionality?

func WithBaseURL

func WithBaseURL(url string) ClientOptionFunc

func WithHeaders

func WithHeaders(headers map[string]string) ClientOptionFunc

func WithLinearAttempt

func WithLinearAttempt(a uint8) ClientOptionFunc

func WithMaxAttempt

func WithMaxAttempt(a uint8) ClientOptionFunc

func WithMaxDelay

func WithMaxDelay(d time.Duration) ClientOptionFunc

func WithMinDelay

func WithMinDelay(d time.Duration) ClientOptionFunc

func WithMultiplier

func WithMultiplier(m float64) ClientOptionFunc

func WithPostResponseHooks

func WithPostResponseHooks(hooks ...ResponseHook) ClientOptionFunc

func WithPreRequestHooks

func WithPreRequestHooks(hooks ...RequestHook) ClientOptionFunc

func WithTimeout

func WithTimeout(timeout time.Duration) ClientOptionFunc

func WithTransformer

func WithTransformer(transformer transform.Transformer) ClientOptionFunc

func WithUserAgentGenerator

func WithUserAgentGenerator(fn func() string) ClientOptionFunc

type Clock

type Clock interface {
	Now() time.Time
	Sleep(d time.Duration)
}

type ExecutableFunc

type ExecutableFunc func() error

type Get

type Get[T any] func() (T, bool)

type LayerFunc

type LayerFunc func(in *goquery.Document, put Put[string])

type Limiter

type Limiter interface {
	Take() (bool, time.Duration)
}

type Logger

type Logger interface {
	Debug(msg string, context ...logContext)
	Info(msg string, context ...logContext)
	Warn(msg string, context ...logContext)
	Error(msg string, context ...logContext)
	Panic(msg string, context ...logContext)
}

type Put

type Put[T any] func(T)

type Remilia

type Remilia struct {
	ID   string
	Name string
	// contains filtered or unexported fields
}

func New

func New(opts ...RemiliaOptionFunc) (*Remilia, error)

func (*Remilia) AddLayer

func (r *Remilia) AddLayer(fn LayerFunc, opts ...StageOptionFunc) actionLayerDef[*Request]

func (*Remilia) Do

func (r *Remilia) Do(pd providerDef[*Request], stageDefs ...actionLayerDef[*Request]) error

func (*Remilia) URLProvider

func (r *Remilia) URLProvider(urlStr string) providerDef[*Request]

type RemiliaOptionFunc

type RemiliaOptionFunc func(*Remilia)

func WithClientOptions

func WithClientOptions(opts ...ClientOptionFunc) RemiliaOptionFunc

func WithLayerOptions

func WithLayerOptions(opts ...StageOptionFunc) RemiliaOptionFunc

func WithLogger

func WithLogger(logger Logger) RemiliaOptionFunc

type Request

type Request struct {
	Method      []byte
	URL         []byte
	Headers     *fasthttp.Args
	Body        []byte
	QueryParams *fasthttp.Args
}

type RequestHook

type RequestHook func(*Request) error

type Response

type Response struct {
	// contains filtered or unexported fields
}

type ResponseHook

type ResponseHook func(*Response) error

type StageOptionFunc

type StageOptionFunc optionFunc[*stageOptions]

func WithConcurrency

func WithConcurrency(concurrency uint) StageOptionFunc

func WithInputBufferSize

func WithInputBufferSize(size uint) StageOptionFunc