output

Documentation

Index

• type APIWriter
  • func NewAPIWriter(wc *WriterConfig) *APIWriter
  • func (f *APIWriter) Write(items chan map[string]interface{}, wg *sync.WaitGroup)
• type FileWriter
  • func NewFileWriter(wc *WriterConfig) *FileWriter
  • func (fr *FileWriter) Write(items chan map[string]interface{}, wg *sync.WaitGroup)
• type StdoutWriter
  • func (s *StdoutWriter) Write(items chan map[string]interface{}, wg *sync.WaitGroup)
• type Writer
• type WriterConfig

Types

type APIWriter

type APIWriter struct {
	// contains filtered or unexported fields
}

The APIWriter is meant to write to a custom API and assumes many things. So currently, it is better not to use this APIWriter.

func NewAPIWriter

func NewAPIWriter(wc *WriterConfig) *APIWriter

NewAPIWriter returns a new APIWriter

func (*APIWriter) Write

func (f *APIWriter) Write(items chan map[string]interface{}, wg *sync.WaitGroup)

type FileWriter

type FileWriter struct {
	// contains filtered or unexported fields
}

func NewFileWriter

func NewFileWriter(wc *WriterConfig) *FileWriter

NewFileWriter returns a new FileWriter

func (*FileWriter) Write

func (fr *FileWriter) Write(items chan map[string]interface{}, wg *sync.WaitGroup)

type StdoutWriter

type StdoutWriter struct{}

func (*StdoutWriter) Write

func (s *StdoutWriter) Write(items chan map[string]interface{}, wg *sync.WaitGroup)

type Writer

type Writer interface {
	// if a writer encounters a fatal error it should call log.Fatalf
	// to prevent the crawler from uselessly continuing to run.
	Write(itemsList chan map[string]interface{}, wg *sync.WaitGroup)
}

type WriterConfig

type WriterConfig struct {
	Type     string `yaml:"type" env:"WRITER_TYPE" env-default:"stdout"`
	Uri      string `yaml:"uri" env:"WRITER_URI"`
	User     string `yaml:"user" env:"WRITER_USER"`
	Password string `yaml:"password" env:"WRITER_PASSWORD"`
	FilePath string `yaml:"filepath" env:"WRITER_FILEPATH"`
}

.WriterConfig defines the necessary paramters to make a new writer which is responsible for writing the scraped data to a specific output eg. stdout.