vautour

Documentation

Index

• Variables
• func Boot(cfg Config)
• type Config
• type Document
  • func NewDocumentFromJSON(s string) (*Document, error)
  • func (d *Document) JSON() string
• type InputModule
• type InputsConfig
• type OutputModule
• type OutputsConfig
• type ProcessedData
• type ProcessorModule
• type ProcessorsConfig
• type QueueModule
• type QueuesConfig
• type ScrapersConfig

Variables

var ErrAlreadyExists = errors.New("document already exists")

Functions

func Boot

func Boot(cfg Config)

Types

type Config

type Config struct {
	Modules map[string]*modules.ModuleConfig

	Inputs     InputsConfig
	Queues     QueuesConfig
	Scrapers   ScrapersConfig
	Processors ProcessorsConfig
	Outputs    OutputsConfig
}

type Document

type Document struct {
	ID      string
	Title   string
	User    string `json:",omitempty"`
	Size    int
	URL     string
	Content []byte

	CreatedAt time.Time
	ExpireAt  time.Time

	Score     int
	Processed []ProcessedData `json:",omitempty"`

	// Internally managed //
	InputModuleName string
}

func NewDocumentFromJSON

func NewDocumentFromJSON(s string) (*Document, error)

func (*Document) JSON

func (d *Document) JSON() string

type InputModule

type InputModule interface {
	Configure(*modules.ModuleConfig) error
	List(*stopper.Stopper, chan *Document) error
	Scrape(*Document) error
}

type InputsConfig

type InputsConfig struct {
	Modules []string
}

type OutputModule

type OutputModule interface {
	Configure(*modules.ModuleConfig) error
	Send(*Document) error
}

type OutputsConfig

type OutputsConfig struct {
	Modules []string
	Threads int
}

type ProcessedData

type ProcessedData struct {
	Module string
	json.RawMessage
}

type ProcessorModule

type ProcessorModule interface {
	Configure(*modules.ModuleConfig) error
	Process(*Document) error
}

type ProcessorsConfig

type ProcessorsConfig struct {
	Modules []string
	Threads int
}

type QueueModule

type QueueModule interface {
	Configure(moduleConfig *modules.ModuleConfig) error
	AddDocument(queue string, d *Document, cacheTTL time.Duration) error
	GetDocument(queue string, ttl time.Duration) (string, *Document, func(time.Duration) error, error)
	ReleaseDocument(queue string, d *Document) error
	DeleteDocument(queue string, d *Document) error
	Bookkeep(queues []string)
}

type QueuesConfig

type QueuesConfig struct {
	Module string
}

type ScrapersConfig

type ScrapersConfig struct {
	Modules []string
	Threads int
}