entity

Documentation

Index

• Variables
• type Cache
  • func NewCache() *Cache
  • func (c *Cache) Get() []byte
  • func (c *Cache) Reader() io.Reader
  • func (c *Cache) Write(p []byte) (n int, err error)
• type File
  • func NewFile(name string, data []byte) File
• type Format
• type Meta
• type Page
  • func NewPage(url string, description string, formats ...Format) *Page
  • func (p *Page) Prepare(ctx context.Context, processor Processor)
  • func (p *Page) Process(ctx context.Context, processor Processor)
  • func (p *Page) SetProcessing()
• type PageBase
• type Pages
• type Processor
• type Result
• type Results
  • func (r *Results) Add(result Result)
  • func (r *Results) MarshalMsgpack() ([]byte, error)
  • func (r *Results) RO() ResultsRO
  • func (r *Results) Results() []Result
  • func (r *Results) UnmarshalMsgpack(b []byte) error
• type ResultsRO
• type Status
• type Worker
  • func NewWorker(ch chan *Page, pages Pages, processor Processor, log *zap.Logger) *Worker
  • func (w *Worker) Start(ctx context.Context, wg *sync.WaitGroup)

Variables

var AllFormats = []Format{
	FormatHeaders,
	FormatPDF,
	FormatSingleFile,
}

Types

type Cache

type Cache struct {
	// contains filtered or unexported fields
}

func NewCache

func NewCache() *Cache

func (*Cache) Get

func (c *Cache) Get() []byte

func (*Cache) Reader

func (c *Cache) Reader() io.Reader

func (*Cache) Write

func (c *Cache) Write(p []byte) (n int, err error)

type File

type File struct {
	ID       uuid.UUID
	Name     string
	MimeType string
	Size     int64
	Data     []byte
	Created  time.Time
}

func NewFile

func NewFile(name string, data []byte) File

type Format

type Format uint8

const (
	FormatHeaders Format = iota
	FormatSingleFile
	FormatPDF
)

type Meta

type Meta struct {
	Title       string
	Description string
	Encoding    string
	Error       string
}

type Page

type Page struct {
	PageBase
	Results ResultsRO
	// contains filtered or unexported fields
}

func NewPage

func NewPage(url string, description string, formats ...Format) *Page

func (*Page) Prepare

func (p *Page) Prepare(ctx context.Context, processor Processor)

func (*Page) Process

func (p *Page) Process(ctx context.Context, processor Processor)

func (*Page) SetProcessing

func (p *Page) SetProcessing()

type PageBase

type PageBase struct {
	ID          uuid.UUID
	URL         string
	Description string
	Created     time.Time
	Formats     []Format
	Version     uint16
	Status      Status
	Meta        Meta
}

type Pages

type Pages interface {
	Save(ctx context.Context, page *Page) error
	ListUnprocessed(ctx context.Context) ([]Page, error)
}

type Processor

type Processor interface {
	Process(ctx context.Context, format Format, url string, cache *Cache) Result
	GetMeta(ctx context.Context, url string, cache *Cache) (Meta, error)
}

type Result

type Result struct {
	Format Format
	Err    error
	Files  []File
}

type Results

type Results struct {
	// contains filtered or unexported fields
}

func (*Results) Add

func (r *Results) Add(result Result)

func (*Results) MarshalMsgpack

func (r *Results) MarshalMsgpack() ([]byte, error)

func (*Results) RO

func (r *Results) RO() ResultsRO

func (*Results) Results

func (r *Results) Results() []Result

func (*Results) UnmarshalMsgpack

func (r *Results) UnmarshalMsgpack(b []byte) error

type ResultsRO

type ResultsRO []Result

type Status

type Status uint8

const (
	StatusNew Status = iota
	StatusProcessing
	StatusDone
	StatusFailed
	StatusWithErrors
)

type Worker

type Worker struct {
	// contains filtered or unexported fields
}

func NewWorker

func NewWorker(ch chan *Page, pages Pages, processor Processor, log *zap.Logger) *Worker

func (*Worker) Start

func (w *Worker) Start(ctx context.Context, wg *sync.WaitGroup)