crawler

package

v0.0.0-...-bdb5213 Latest Latest Go to latest Published: Oct 12, 2024 License: MIT Imports: 14 Imported by: 0

Details

This section is empty.

This section is empty.

This section is empty.

type Client interface {
	Get(ctx context.Context, url *url.URL) (string, error)
}

type Config struct {
	NumWorkers       int
	BaseURL          url.URL
	Resume           bool
	DownloadPathBase string
	Debug            bool
}

type ContentRepository interface {
	Save(path string, content string) error
	Exists(path string) bool
	GetData(path string) (string, error)
}

type FileSystemRepository struct{}

func NewFileSystemRepository() *FileSystemRepository

func (f *FileSystemRepository) Exists(path string) bool

func (f *FileSystemRepository) GetData(path string) (string, error)

func (f *FileSystemRepository) Save(path string, content string) error

type HTTPClient struct {
	// contains filtered or unexported fields
}

func NewHTTPClient(cfg *Config) *HTTPClient

func (h *HTTPClient) Get(ctx context.Context, url *url.URL) (string, error)

type ParseResult struct {
	URLs []string
}

type Parser interface {
	Parse(contents string) (*ParseResult, error)
}

type Queue[T any] interface {
	Put(item T)
	Get() T
	Out() <-chan T
	Close()
}

type SimpleParser struct{}

func NewParser() *SimpleParser

func (p *SimpleParser) Parse(contents string) (*ParseResult, error)

type UnboundedQueue[T any] struct {
	// contains filtered or unexported fields
}

func NewUnboundedQueue[T any]() *UnboundedQueue[T]

func (q *UnboundedQueue[T]) Close()

func (q *UnboundedQueue[T]) Get() T

func (q *UnboundedQueue[T]) Out() <-chan T

func (q *UnboundedQueue[T]) Put(item T)

type WorkerPool struct {
	// contains filtered or unexported fields
}

func NewWorkerPool(
	cfg *Config,
	queue Queue[string],
	parser Parser,
	contentRepository ContentRepository,
	client Client,
) *WorkerPool

func (w *WorkerPool) Start(ctx context.Context) error

func (w *WorkerPool) WaitOn()