kafka

Documentation

Index

• Variables
• type KafkaPartitionInfo
  • func (s *KafkaPartitionInfo) ReadSplit() error
• type KafkaSource
  • func New(brokers []string, topic, group string) *KafkaSource
  • func (s *KafkaSource) Generate(f *flow.Flow) *flow.Dataset
  • func (s *KafkaSource) Timeout(seconds int) *KafkaSource

Variables

var (
	MapperReadShard = gio.RegisterMapper(readShard)
)

Types

type KafkaPartitionInfo

type KafkaPartitionInfo struct {
	Brokers        []string
	Topic          string
	Group          string
	TimeoutSeconds int
	PartitionId    int32
}

func (*KafkaPartitionInfo) ReadSplit

func (s *KafkaPartitionInfo) ReadSplit() error

type KafkaSource

type KafkaSource struct {
	Brokers        []string
	Group          string
	Topic          string
	TimeoutSeconds int
	// contains filtered or unexported fields
}

func New

func New(brokers []string, topic, group string) *KafkaSource

func (*KafkaSource) Generate

func (s *KafkaSource) Generate(f *flow.Flow) *flow.Dataset

Generate generates data shard info, partitions them via round robin, and reads each shard on each executor

func (*KafkaSource) Timeout

func (s *KafkaSource) Timeout(seconds int) *KafkaSource