remilia

package module

Go to main page

Versions in this module

v0

v0.5.5

Jun 30, 2024

Changes in this version

+ type Bucket struct

+ func NewBucket(clock Clock, capacity int64, fillInterval time.Duration, fillQuantum int64, ...) *Bucket

+ func (b *Bucket) Take(count int64) time.Duration

+ func (b *Bucket) Wrap(op func() error) ExecutableFunc

+ type Clock interface

+ Now func() time.Time

+ Sleep func(d time.Duration)

+ type ExecutableFunc func() error

+ type LayerFunc func(in *goquery.Document, put Put[string])

+ type Limiter interface

+ Take func() (bool, time.Duration)

type Remilia

+ func (r *Remilia) AddLayer(fn LayerFunc, opts ...StageOptionFunc) actionLayerDef[*Request]

+ func (r *Remilia) URLProvider(urlStr string) providerDef[*Request]

+ type RemiliaOptionFunc func(*Remilia)

+ func WithLayerOptions(opts ...StageOptionFunc) RemiliaOptionFunc

+ type StageOptionFunc optionFunc[*stageOptions]

v0.5.4

May 1, 2024

Changes in this version

+ func WithWorkLinearAttempt(a uint8) exponentialBackoffOptionFunc

+ func WithWorkMaxAttempt(a uint8) exponentialBackoffOptionFunc

+ func WithWorkMaxDelay(d time.Duration) exponentialBackoffOptionFunc

+ func WithWorkMinDelay(d time.Duration) exponentialBackoffOptionFunc

+ func WithWorkMultiplier(m float64) exponentialBackoffOptionFunc

v0.5.2

Apr 27, 2024

Changes in this version

+ func WithApiKeyAuth(apiKey string) clientOptionFunc

+ func WithBasicAuth(username string, password string) clientOptionFunc

+ func WithBearerAuth(token string) clientOptionFunc

+ func WithCookie(cookie string) clientOptionFunc

+ func WithRequestLinearAttempt(a uint8) clientOptionFunc

+ func WithRequestMaximumAttempt(a uint8) clientOptionFunc

+ func WithRequestMaximumDelay(d time.Duration) clientOptionFunc

+ func WithRequestMinimumDelay(d time.Duration) clientOptionFunc

+ func WithRequestMultiplier(m float64) clientOptionFunc

+ func WithUnitOptions(opts ...unitOptionFunc) remiliaOption

+ func WithUserAgentGenerator(fn func() string) clientOptionFunc

v0.5.1

Feb 16, 2024

v0.5.0

Feb 16, 2024

Changes in this version

+ func WithBackoffOptions(opts ...exponentialBackoffOptionFunc) remiliaOption

+ func WithClientOptions(opts ...clientOptionFunc) remiliaOption

+ func WithConcurrency(concurrency uint) stageOptionFunc

+ func WithInputBufferSize(size uint) stageOptionFunc

+ func WithStageOptions(opts ...stageOptionFunc) remiliaOption

+ func WithTransformer(transformer transform.Transformer) clientOptionFunc

+ type UnitFunc func(in *goquery.Document, put Put[string])

v0.4.0

Feb 15, 2024

v0.3.0

Feb 14, 2024

Changes in this version

+ var DefaultLinearAttempt = uint8(5)

+ var DefaultMaxAttempt = uint8(10)

+ var DefaultMaxDelay = 10 * time.Second

+ var DefaultMinDelay = 100 * time.Millisecond

+ var DefaultMultiplier = 2.0

+ var DefaultRandom = &defaultRandom

+ func Retry(ctx context.Context, op RetryableFunc, eb Backoff) error

+ type Backoff interface

+ GetCurrentAttempt func() uint8

+ GetMaxAttempt func() uint8

+ Next func() time.Duration

+ Reset func()

+ type BatchGetFunc func() ([]T, error)

+ type ClientOptionFunc OptionFunc[*Client]

+ func WithBackoffPool(backoffPool *Pool[*ExponentialBackoff]) ClientOptionFunc

+ func WithBaseURL(url string) ClientOptionFunc

+ func WithClientLogger(logger Logger) ClientOptionFunc

+ func WithDocumentCreator(creator DocumentCreator) ClientOptionFunc

+ func WithHeaders(headers map[string]string) ClientOptionFunc

+ func WithInternalClient(client InternalClient) ClientOptionFunc

+ func WithInternalPostResponseHooks(hooks ...ResponseHook) ClientOptionFunc

+ func WithInternalPreRequestHooks(hooks ...RequestHook) ClientOptionFunc

+ func WithPostResponseHooks(hooks ...ResponseHook) ClientOptionFunc

+ func WithPreRequestHooks(hooks ...RequestHook) ClientOptionFunc

+ func WithReaderPool(readerPool *Pool[*bytes.Reader]) ClientOptionFunc

+ func WithTimeout(timeout time.Duration) ClientOptionFunc

+ type ExponentialBackoff struct

+ func NewExponentialBackoff(optFns ...ExponentialBackoffOptionFunc) *ExponentialBackoff

+ func (eb *ExponentialBackoff) GetCurrentAttempt() uint8

+ func (eb *ExponentialBackoff) GetMaxAttempt() uint8

+ func (eb *ExponentialBackoff) Next() time.Duration

+ func (eb *ExponentialBackoff) Reset()

+ type ExponentialBackoffFactory struct

+ func NewExponentialBackoffFactory(opts ...ExponentialBackoffOptionFunc) *ExponentialBackoffFactory

+ func (f *ExponentialBackoffFactory) New() *ExponentialBackoff

+ func (f *ExponentialBackoffFactory) Reset(eb *ExponentialBackoff)

+ type ExponentialBackoffOptionFunc OptionFunc[*ExponentialBackoff]

+ func WithLinearAttempt(a uint8) ExponentialBackoffOptionFunc

+ func WithMaxAttempt(a uint8) ExponentialBackoffOptionFunc

+ func WithMaxDelay(d time.Duration) ExponentialBackoffOptionFunc

+ func WithMinDelay(d time.Duration) ExponentialBackoffOptionFunc

+ func WithMultiplier(m float64) ExponentialBackoffOptionFunc

+ func WithRandomImp(r Random) ExponentialBackoffOptionFunc

+ type Factory interface

+ New func() T

+ Reset func(T)

+ type JitterBackoff func(attempt uint8) time.Duration

+ func FullJitterBuilder(minDelay time.Duration, capacity time.Duration, multiplier float64, ...) JitterBackoff

+ type OptionFunc func(ins T) error

+ type Random interface

+ Int63n func(n int64) int64

+ type ReaderFactory struct

+ func (f ReaderFactory) New() *bytes.Reader

+ func (f ReaderFactory) Reset(r *bytes.Reader)

type Remilia

+ func (r *Remilia) Unit(fn func(in *goquery.Document, put Put[string], chew Put[string])) StageDef[*Request]

+ type RetryableFunc func() error

+ type StageDef func() (*stage[T], error)

+ func NewStage[T any](fn StageFunc[T], optFns ...StageOptionFn) StageDef[T]

+ type StageFunc func(get Get[T], put Put[T], chew Put[T], inCh chan T) error

v0.2.0

Feb 11, 2024

Changes in this version

+ var ErrInvalidConcurrency = errors.New("invalid concurrency")

+ var ErrInvalidInputBufferSize = errors.New("invalid input buffer size")

+ var ErrInvalidTimeout = errors.New("invalid timeout")

+ func FanIn[T any](done <-chan struct{}, channels ...<-chan T) <-chan T

+ func GetOrDefault(s *string, def string) string

+ func Tee[T any](ctx context.Context, input <-chan T, wg *sync.WaitGroup) (<-chan T, <-chan T)

+ func URLMatcher() func(s string) bool

+ type Client struct

+ func NewClient(client InternalClient, docCreator DocumentCreator, optFns ...ClientOptionFn) (*Client, error)

+ func (c *Client) Execute(request *Request) (*Response, error)

+ type ClientOptionFn OptionFn[*clientOptions]

+ func BaseURL(url string) ClientOptionFn

+ func ClientLogger(logger Logger) ClientOptionFn

+ func Headers(headers map[string]string) ClientOptionFn

+ func InternalPostResponseHooks(hooks ...ResponseHook) ClientOptionFn

+ func InternalPreRequestHooks(hooks ...RequestHook) ClientOptionFn

+ func PostResponseHooks(hooks ...ResponseHook) ClientOptionFn

+ func PreRequestHooks(hooks ...RequestHook) ClientOptionFn

+ func Timeout(timeout time.Duration) ClientOptionFn

+ type DefaultDocumentCreator struct

+ func (d DefaultDocumentCreator) NewDocumentFromReader(r io.Reader) (*goquery.Document, error)

+ type DefaultLogger struct

+ func (l *DefaultLogger) Debug(msg string, context ...LogContext)

+ func (l *DefaultLogger) Error(msg string, context ...LogContext)

+ func (l *DefaultLogger) Info(msg string, context ...LogContext)

+ func (l *DefaultLogger) Panic(msg string, context ...LogContext)

+ func (l *DefaultLogger) Warn(msg string, context ...LogContext)

+ type DocumentCreator interface

+ NewDocumentFromReader func(io.Reader) (*goquery.Document, error)

+ type FileSystem struct

+ func (fs FileSystem) MkdirAll(path string, perm os.FileMode) error

+ func (fs FileSystem) OpenFile(name string, flag int, perm os.FileMode) (*os.File, error)

+ type FileSystemOperations interface

+ MkdirAll func(path string, perm os.FileMode) error

+ OpenFile func(name string, flag int, perm os.FileMode) (*os.File, error)

+ type FlowDef func() (*flow[T], error)

+ func NewFlow[T any](fn FlowFn[T], optFns ...StageOptionFn) FlowDef[T]

+ type FlowFn func(in T) (out T, err error)

+ type Get func() (T, bool)

+ type HTTPClient interface

+ Execute func(request *Request) (*Response, error)

+ type InternalClient interface

+ Do func(req *fasthttp.Request, resp *fasthttp.Response) error

+ type Limit struct

+ AllowedDomain string

+ Delay time.Duration

+ func NewLimit(domain string, delay time.Duration) *Limit

+ type LogContext map[string]interface

+ type LogLevel int8

+ const DebugLevel

+ const ErrorLevel

+ const InfoLevel

+ const WarnLevel

+ type Logger interface

+ Debug func(msg string, context ...LogContext)

+ Error func(msg string, context ...LogContext)

+ Info func(msg string, context ...LogContext)

+ Panic func(msg string, context ...LogContext)

+ Warn func(msg string, context ...LogContext)

+ type LoggerConfig struct

+ ConsoleLevel LogLevel

+ FileLevel LogLevel

+ ID string

+ Name string

+ type MockDocumentCreator struct

+ Doc *goquery.Document

+ Err error

+ func (d MockDocumentCreator) NewDocumentFromReader(r io.Reader) (*goquery.Document, error)

+ type MockFileSystem struct

+ MkdirAllErr error

+ OpenFileErr error

+ OpenFileMock *os.File

+ func (mfs MockFileSystem) MkdirAll(path string, perm os.FileMode) error

+ func (mfs MockFileSystem) OpenFile(name string, flag int, perm os.FileMode) (*os.File, error)

+ type MockHTTPClient struct

+ func (m *MockHTTPClient) Execute(request *Request) (*Response, error)

+ type MockInternalClient struct

+ func (m *MockInternalClient) Do(req *fasthttp.Request, resp *fasthttp.Response) error

+ type MockLogger struct

+ func (m *MockLogger) Debug(msg string, context ...LogContext)

+ func (m *MockLogger) Error(msg string, context ...LogContext)

+ func (m *MockLogger) Info(msg string, context ...LogContext)

+ func (m *MockLogger) Panic(msg string, context ...LogContext)

+ func (m *MockLogger) Warn(msg string, context ...LogContext)

type Option

+ func WithClient(client HTTPClient) Option

+ func WithDefaultClient() Option

+ func WithDefaultLogger() Option

+ func WithLogger(logger Logger) Option

+ type OptionFn func(ins T) error

+ type Pool struct

+ func NewPool[T any](newFunc func() T) *Pool[T]

+ func (p *Pool[T]) Get() T

+ func (p *Pool[T]) Put(item T)

+ type ProcessorDef func() (*processor[T], error)

+ func NewProcessor[T any](fn WorkFn[T], optFns ...StageOptionFn) ProcessorDef[T]

+ type Put func(T)

type Remilia

+ func (r *Remilia) Do(producerDef ProcessorDef[*Request], stageDefs ...ProcessorDef[*Request]) error

+ func (r *Remilia) Just(urlStr string) ProcessorDef[*Request]

+ func (r *Remilia) Relay(fn func(in *goquery.Document, put Put[string])) ProcessorDef[*Request]

+ func (r *Remilia) Sink(fn func(in *goquery.Document) error) FlowDef[*Request]

+ type Request struct

+ Body []byte

+ Headers map[string]string

+ Method string

+ QueryParams map[string]string

+ URL string

+ func EmptyRequest() *Request

+ func NewRequest(opts ...RequestOption) (*Request, error)

+ func (req *Request) Build() *fasthttp.Request

+ type RequestHook func(*Client, *Request) error

+ type RequestOption func(*Request) error

+ func WithBody(body []byte) RequestOption

+ func WithHeader(key, value string) RequestOption

+ func WithMethod(method string) RequestOption

+ func WithQueryParam(key, value string) RequestOption

+ func WithURL(url string) RequestOption

+ type Response struct

+ type ResponseHook func(*Client, *Response) error

+ type StageOptionFn OptionFn[*stageOptions]

+ func Concurrency(concurrency uint) StageOptionFn

+ func InputBufferSize(size uint) StageOptionFn

+ type WorkFn func(get Get[T], put Put[T], chew Put[T]) error

v0.1.0

Aug 26, 2023

Changes in this version

+ type DataConsumer func(data <-chan interface{})

+ type HTMLProcessor struct

+ DataConsumer DataConsumer

+ Fn func(s *goquery.Selection) interface{}

+ Selector string

+ type Middleware struct

+ type Option interface

+ func AllowedDomains(domains ...string) Option

+ func ConcurrentNumber(num int) Option

+ func Delay(delay time.Duration) Option

+ func Name(name string) Option

+ func UserAgent(ua string) Option

+ type Remilia struct

+ AllowedDomains []string

+ ConcurrentNumber int

+ Delay time.Duration

+ ID string

+ Name string

+ URL string

+ UserAgent string

+ func New(url string, options ...Option) *Remilia

+ func (r *Remilia) AddToChain() *Remilia

+ func (r *Remilia) Start() error

+ func (r *Remilia) UseHTML(selector string, htmlProcFn func(s *goquery.Selection) interface{}, ...) *Remilia

+ func (r *Remilia) UseURL(selector string, urlGenFn func(s *goquery.Selection) *url.URL) *Remilia

+ type URLGenerator struct

+ Fn func(s *goquery.Selection) *url.URL

+ Selector string

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL