worker

Documentation

Index

• func NewMapper(resource *model.Resource) mapreduce.Mapper
• func NewReducer(s store.Store) mapreduce.Reducer
• func NewShuffler(size int) mapreduce.Shuffler
• type Mapper
  • func (m *Mapper) Map(ctx context.Context, rows <-chan []string, entities chan<- mapreduce.Entity) error
• type Reducer
  • func (r *Reducer) Reduce(ctx context.Context, batches <-chan []mapreduce.Entity) error
• type Shuffler
  • func (s *Shuffler) Shuffle(ctx context.Context, entities <-chan mapreduce.Entity, ...) error
• type Worker
  • func New(s store.Store) *Worker
  • func (w *Worker) ModifiedResources() (map[string]time.Time, error)
  • func (w *Worker) Process(ctx context.Context, resources <-chan govdata.Resource) error

Functions

func NewMapper

func NewMapper(resource *model.Resource) mapreduce.Mapper

func NewReducer

func NewReducer(s store.Store) mapreduce.Reducer

func NewShuffler

func NewShuffler(size int) mapreduce.Shuffler

Types

type Mapper

type Mapper struct {
	// contains filtered or unexported fields
}

func (*Mapper) Map

func (m *Mapper) Map(ctx context.Context, rows <-chan []string, entities chan<- mapreduce.Entity) error

type Reducer

type Reducer struct {
	// contains filtered or unexported fields
}

func (*Reducer) Reduce

func (r *Reducer) Reduce(ctx context.Context, batches <-chan []mapreduce.Entity) error

type Shuffler

type Shuffler struct {
	// contains filtered or unexported fields
}

func (*Shuffler) Shuffle

func (s *Shuffler) Shuffle(ctx context.Context, entities <-chan mapreduce.Entity, batches chan<- []mapreduce.Entity) error

type Worker

type Worker struct {
	// contains filtered or unexported fields
}

Worker is responsible for processing incoming data.

func New

func New(s store.Store) *Worker

New returns new instance of worker.

func (*Worker) ModifiedResources

func (w *Worker) ModifiedResources() (map[string]time.Time, error)

func (*Worker) Process

func (w *Worker) Process(ctx context.Context, resources <-chan govdata.Resource) error

Process dispatches event as to a handler.