storage

Documentation

Index

• Constants
• func WriteLinksToFile(links map[string]struct{}, filename string) error
• func WriteLinksToFileRandomFilename(links map[string]struct{}, baseDir string) error
• type NSQDServer
  • func NewNSQDServer() (*NSQDServer, error)
  • func (s *NSQDServer) Error() <-chan error
  • func (s *NSQDServer) Stop()
• type Queue
  • func NewQueue(path string, workerCount int) (*Queue, error)
  • func (store *Queue) AddURL(targetURL string) error
  • func (store *Queue) CloseDB() error
  • func (store *Queue) IsStopped() bool
  • func (store *Queue) StopProducer()
  • func (store *Queue) StopSignal()

Constants

const NsqChannel = "channel"

const NsqServer = "localhost:4150"

const NsqTopic = "scraper_"

Functions

func WriteLinksToFile

func WriteLinksToFile(links map[string]struct{}, filename string) error

func WriteLinksToFileRandomFilename

func WriteLinksToFileRandomFilename(links map[string]struct{}, baseDir string) error

Types

type NSQDServer

type NSQDServer struct {
	// contains filtered or unexported fields
}

func NewNSQDServer

func NewNSQDServer() (*NSQDServer, error)

func (*NSQDServer) Error

func (s *NSQDServer) Error() <-chan error

func (*NSQDServer) Stop

func (s *NSQDServer) Stop()

type Queue

type Queue struct {
	// contains filtered or unexported fields
}

func NewQueue

func NewQueue(path string, workerCount int) (*Queue, error)

func (*Queue) AddURL

func (store *Queue) AddURL(targetURL string) error

TODO: Canonicalize links before everything

func (*Queue) CloseDB

func (store *Queue) CloseDB() error

func (*Queue) IsStopped

func (store *Queue) IsStopped() bool

func (*Queue) StopProducer

func (store *Queue) StopProducer()

func (*Queue) StopSignal

func (store *Queue) StopSignal()