xlget

Documentation

Overview

Package xlget implements a blacklist downloader.

This package is a work in progress and makes no API stability promises.

Index

• Variables
• func SetHTTPClient(h *grab.Client)
• func ValidCompression(c Compression) bool
• func ValidFormatSource(f FormatSource) bool
• func ValidURI(uri string) bool
• func ValidateEntry(e Entry) error
• type AccountItem
• type CancelFunc
• type Client
  • func NewClient(cfg Config) *Client
  • func (c *Client) Do(e Entry) (*Response, error)
• type Compression
  • func (c Compression) MarshalJSON() ([]byte, error)
  • func (c Compression) String() string
  • func (c *Compression) UnmarshalJSON(data []byte) error
• type Config
• type Duration
  • func (d Duration) MarshalJSON() (b []byte, err error)
  • func (d *Duration) UnmarshalJSON(b []byte) (err error)
• type Entry
  • func EntryDefsFromFile(path string) ([]Entry, error)
  • func (e Entry) Copy() (dst Entry)
• type EntryStatus
  • func EntryStatusFromFile(path string) (EntryStatus, error)
• type FormatOpts
• type FormatSource
  • func (f FormatSource) MarshalJSON() ([]byte, error)
  • func (f FormatSource) String() string
  • func (f *FormatSource) UnmarshalJSON(data []byte) error
• type Manager
  • func NewManager(outputDir, cacheDir, statusDir string, opt ...Option) (*Manager, error)
  • func (m *Manager) Add(entries []Entry) error
  • func (m *Manager) Clear()
  • func (m *Manager) Deprecated() []string
  • func (m *Manager) NeedsUpdate() []string
  • func (m *Manager) Removed() []string
  • func (m *Manager) Update() (CancelFunc, <-chan struct{}, error)
• type Option
  • func SetLogger(l yalogi.Logger) Option
• type Response
  • func (r *Response) Cancel() bool
  • func (r *Response) Err() error
  • func (r *Response) IsComplete() bool
  • func (r *Response) Status() State
  • func (r *Response) Wait() error
• type Source
  • func (s Source) Copy() (dst Source)
• type State
• type TransformOpts

Variables

var (
	ErrCanceled = errors.New("download canceled")
)

Common errors

Functions

func SetHTTPClient

func SetHTTPClient(h *grab.Client)

SetHTTPClient sets http client

func ValidCompression

func ValidCompression(c Compression) bool

ValidCompression returns true if is valid compression.

func ValidFormatSource

func ValidFormatSource(f FormatSource) bool

ValidFormatSource returns true if is a valid format.

func ValidURI

func ValidURI(uri string) bool

ValidURI returns true if is a valid uri for downloader.

func ValidateEntry

func ValidateEntry(e Entry) error

ValidateEntry checks if entry is valid

Types

type AccountItem

type AccountItem struct {
	Resource xlist.Resource `json:"resource"`
	Count    int            `json:"count"`
}

AccountItem stores accounting info I can't use a simple map[xlist.Resources] because this issue: //https://github.com/golang/go/issues/29732

type CancelFunc

type CancelFunc func()

CancelFunc defines a type for cancelation function

type Client

type Client struct {
	// contains filtered or unexported fields
}

Client downloads requests

func NewClient

func NewClient(cfg Config) *Client

NewClient returns a new Client

func (*Client) Do

func (c *Client) Do(e Entry) (*Response, error)

Do request a download

type Compression

type Compression int

Compression defines compression algs

const (
	None Compression = iota
	Gzip
	Zip
)

List of compression values

func (Compression) MarshalJSON

func (c Compression) MarshalJSON() ([]byte, error)

MarshalJSON implements interface for struct marshalling.

func (Compression) String

func (c Compression) String() string

func (*Compression) UnmarshalJSON

func (c *Compression) UnmarshalJSON(data []byte) error

UnmarshalJSON implements interface for struct unmarshalling.

type Config

type Config struct {
	OutputDir string
	CacheDir  string
	Logger    yalogi.Logger
}

Config defines configuration.

type Duration

type Duration struct {
	time.Duration
}

Duration is used for unmarshalling durations. tip from: https://robreid.io/json-time-duration/

func (Duration) MarshalJSON

func (d Duration) MarshalJSON() (b []byte, err error)

MarshalJSON implements interface

func (*Duration) UnmarshalJSON

func (d *Duration) UnmarshalJSON(b []byte) (err error)

UnmarshalJSON implements interface

type Entry

type Entry struct {
	ID         string         `json:"id"`
	Disabled   bool           `json:"disabled,omitempty"`
	Removed    bool           `json:"removed,omitempty"`
	Deprecated bool           `json:"deprecated,omitempty"`
	Update     Duration       `json:"update"`
	Sources    []Source       `json:"sources"`
	Transforms *TransformOpts `json:"transforms,omitempty"`
	NoClean    bool           `json:"noclean,omitempty"`
	NoHash     bool           `json:"nohash,omitempty"`
	Output     string         `json:"output,omitempty"`
}

Entry defines configuration entries format

func EntryDefsFromFile

func EntryDefsFromFile(path string) ([]Entry, error)

EntryDefsFromFile returns an Entry slice of configuration

func (Entry) Copy

func (e Entry) Copy() (dst Entry)

Copy returns a copy of Entry.

type EntryStatus

type EntryStatus struct {
	ID string `json:"id"`
	// First sync and last sync
	First time.Time `json:"first"`
	Last  time.Time `json:"last"`
	// Updates stores only successful syncs
	Updates    int        `json:"updates"`
	LastUpdate *time.Time `json:"lastupdate,omitempty"`
	// Changes stores changes in list (md5 changed)
	Changes    int        `json:"changes"`
	LastChange *time.Time `json:"lastchange,omitempty"`
	// Errors stores number of sync errors (not errors in file)
	Errors    int        `json:"errors"`
	LastError *time.Time `json:"lasterror,omitempty"`
	// Last sync state
	UpdatedOK bool          `json:"updatedok"`
	Account   []AccountItem `json:"account,omitempty"`
	ErrorMsg  string        `json:"errormsg,omitempty"`
}

EntryStatus stores status information

func EntryStatusFromFile

func EntryStatusFromFile(path string) (EntryStatus, error)

EntryStatusFromFile returns an EntryStatus

type FormatOpts

type FormatOpts struct {
	Comma      string `json:"comma,omitempty"`
	Comment    string `json:"comment,omitempty"`
	HasHeader  bool   `json:"header,omitempty"`
	Indexes    []int  `json:"indexes,omitempty"`
	LazyQuotes bool   `json:"lazyquotes,omitempty"`
}

FormatOpts defines format options for conversors

type FormatSource

type FormatSource int

FormatSource defines source formats

const (
	XList FormatSource = iota
	Flat
	CSV
	Hosts
)

List of source format values

func (FormatSource) MarshalJSON

func (f FormatSource) MarshalJSON() ([]byte, error)

MarshalJSON implements interface for struct marshalling.

func (FormatSource) String

func (f FormatSource) String() string

func (*FormatSource) UnmarshalJSON

func (f *FormatSource) UnmarshalJSON(data []byte) error

UnmarshalJSON implements interface for struct unmarshalling.

type Manager

type Manager struct {
	// contains filtered or unexported fields
}

Manager processes configuration entries and checks for required updates

func NewManager

func NewManager(outputDir, cacheDir, statusDir string, opt ...Option) (*Manager, error)

NewManager creates a new manager

func (*Manager) Add

func (m *Manager) Add(entries []Entry) error

Add entries to manager

func (*Manager) Clear

func (m *Manager) Clear()

Clear entries

func (*Manager) Deprecated

func (m *Manager) Deprecated() []string

Deprecated returns an slice with ids deprecated

func (*Manager) NeedsUpdate

func (m *Manager) NeedsUpdate() []string

NeedsUpdate returns an slice with ids that must be updated

func (*Manager) Removed

func (m *Manager) Removed() []string

Removed returns an slice with ids removed

func (*Manager) Update

func (m *Manager) Update() (CancelFunc, <-chan struct{}, error)

Update entries registered in backbround, it returns a cancelation function, a done channel (it will close when process done) and an error.

type Option

type Option func(*options)

Option is used for manager configuration

func SetLogger

func SetLogger(l yalogi.Logger) Option

SetLogger option allows set a custom logger

type Response

type Response struct {
	ID         string
	Done       chan struct{}
	Start, End time.Time
	Account    map[xlist.Resource]int
	Updated    bool
	Output     string
	Hash       string
	// contains filtered or unexported fields
}

Response stores information about download and conversion

func (*Response) Cancel

func (r *Response) Cancel() bool

Cancel allows cancel process

func (*Response) Err

func (r *Response) Err() error

Err returns stored error

func (*Response) IsComplete

func (r *Response) IsComplete() bool

IsComplete returns true if completed

func (*Response) Status

func (r *Response) Status() State

Status returns current state value

func (*Response) Wait

func (r *Response) Wait() error

Wait for finish

type Source

type Source struct {
	URI         string      `json:"uri"`
	Filename    string      `json:"filename,omitempty"`
	Compression Compression `json:"compression,omitempty"`

	Format     FormatSource     `json:"format"`
	FormatOpts *FormatOpts      `json:"formatopts,omitempty"`
	Resources  []xlist.Resource `json:"resources,omitempty"`
	Limit      int              `json:"limit,omitempty"`
}

Source defines configuration for sources

func (Source) Copy

func (s Source) Copy() (dst Source)

Copy returns a copy of Source.

type State

type State int

State type defines available states for Response

const (
	Ready State = iota
	Downloading
	Uncompressing
	Converting
	Deploying
	Cleaning
	Finished
)

List of States

type TransformOpts

type TransformOpts struct {
	TLDPlusOne bool `json:"tldplusone,omitempty"`
}

TransformOpts defines transformations.