data

Documentation

Index

• Constants
• func AbsAtomPath() (string, error)
• func AbsCookiesPath() (string, error)
• func AbsSourcesPath() (string, error)
• func AllAbsDirs() []pathways.AbsDir
• func AllProperties() []string
• func LoadRecipes() (wits.SectionKeyValue, error)
• func SourcesIdPropertyValues(sources []*Source) map[string]map[string][]string
• func SourcesIds(src ...*Source) []string
• type QuerySelectors
  • func NewQuerySelectors(kv wits.KeyValue) *QuerySelectors
  • func (qs *QuerySelectors) IsValid() error
  • func (qs *QuerySelectors) Override(another *QuerySelectors)
• type Source
  • func LoadSources() ([]*Source, error)
  • func SourceById(id string, src ...*Source) *Source
  • func (src *Source) IsValid() error

Constants

const (
	Backups        pathways.AbsDir = "backups"
	Input          pathways.AbsDir = "input"
	LocalContent   pathways.AbsDir = "local-content"
	MatchedContent pathways.AbsDir = "matched-content"
	Output         pathways.AbsDir = "output"
	Redux          pathways.AbsDir = "redux"
)

const (
	GetContentErrorsProperty   = "get-content-errors"
	DecodeErrorsProperty       = "decode-errors"
	MatchContentErrorsProperty = "match-content-errors"
	ReduceErrorsProperty       = "reduce-errors"

	CurrentElementsProperty  = "current-elements"
	AddedElementsProperty    = "added-elements"
	RemovedElementsProperty  = "removed-elements"
	PreviousElementsProperty = "previous-elements"
	SourceURLProperty        = "source-url"
)

const (
	Id                       = "id"
	URL                      = "url"
	Recipe                   = "recipe"
	Encoding                 = "encoding"
	Title                    = "title"
	Category                 = "category"
	ContainerSelector        = "container-selector"
	TextContent              = "text-content"
	ElementsSelector         = "elements-selector"
	ElementReductionSelector = "element-reduction-selector"
	ElementAttribute         = "element-attribute"
)

const DefaultNovusRootDir = "/usr/share/novus"

Functions

func AbsAtomPath

func AbsAtomPath() (string, error)

func AbsCookiesPath

func AbsCookiesPath() (string, error)

func AbsSourcesPath

func AbsSourcesPath() (string, error)

func AllAbsDirs

func AllAbsDirs() []pathways.AbsDir

func AllProperties

func AllProperties() []string

func LoadRecipes

func LoadRecipes() (wits.SectionKeyValue, error)

func SourcesIdPropertyValues

func SourcesIdPropertyValues(sources []*Source) map[string]map[string][]string

func SourcesIds

func SourcesIds(src ...*Source) []string

Types

type QuerySelectors

type QuerySelectors struct {
	ContainerSelector        string
	TextContent              string
	ElementsSelector         string
	ElementReductionSelector string
	ElementAttribute         string
}

func NewQuerySelectors

func NewQuerySelectors(kv wits.KeyValue) *QuerySelectors

func (*QuerySelectors) IsValid

func (qs *QuerySelectors) IsValid() error

func (*QuerySelectors) Override

func (qs *QuerySelectors) Override(another *QuerySelectors)

type Source

type Source struct {
	Id       string
	URL      *url.URL
	Recipe   string
	Encoding string
	Title    string
	Category string
	Query    *QuerySelectors
}

func LoadSources

func LoadSources() ([]*Source, error)

func SourceById

func SourceById(id string, src ...*Source) *Source

func (*Source) IsValid

func (src *Source) IsValid() error