bundledImpl

Documentation

Index

• func FullDestinPath(destDir string, fullSourcePath string, extension string) string
• type DummyEvent
  • func (de *DummyEvent) Process(reader io.Reader, customParams interface{}) bool
  • func (de *DummyEvent) Setup(customParams interface{}) bool
• type FetchAndExecEvent
  • func (fae *FetchAndExecEvent) Process(reader io.Reader, customParams interface{}) bool
  • func (fae *FetchAndExecEvent) Setup(customParams interface{}) bool
• type KafkaEmitEvent
  • func (kee *KafkaEmitEvent) Process(reader io.Reader, customParams interface{}) bool
  • func (kee *KafkaEmitEvent) Setup(customParams interface{}) bool
• type S3
  • func NewS3() *S3
  • func (s3 S3) GetFileSet(minioclient *minio.Client) map[string]int64
  • func (s3 S3) GetFilteredFileSet(filter string, minioClient *minio.Client) map[string]int64
  • func (s3 S3) ListFiles(minioClient *minio.Client) map[string]api.FileEntity
  • func (s3 *S3) Watch(eventHandlers []api.IOEvent) bool

Functions

func FullDestinPath

func FullDestinPath(destDir string, fullSourcePath string, extension string) string

extension example strconv.Itoa(key)+."parquet"

Types

type DummyEvent

type DummyEvent struct {
}

func (*DummyEvent) Process

func (de *DummyEvent) Process(reader io.Reader, customParams interface{}) bool

func (*DummyEvent) Setup

func (de *DummyEvent) Setup(customParams interface{}) bool

type FetchAndExecEvent

type FetchAndExecEvent struct {
	DestinationFolder string
	CmdTemplate       string
	Data              map[string]interface{}
	Tmpl              *template.Template
}

func (*FetchAndExecEvent) Process

func (fae *FetchAndExecEvent) Process(reader io.Reader, customParams interface{}) bool

func (*FetchAndExecEvent) Setup

func (fae *FetchAndExecEvent) Setup(customParams interface{}) bool

type KafkaEmitEvent

type KafkaEmitEvent struct {
	Data     map[string]interface{}
	Tmpl     *template.Template
	Producer *kafka.Producer
	Topic    string
}

func (*KafkaEmitEvent) Process

func (kee *KafkaEmitEvent) Process(reader io.Reader, customParams interface{}) bool

func (*KafkaEmitEvent) Setup

func (kee *KafkaEmitEvent) Setup(customParams interface{}) bool

type S3

type S3 struct {
	Endpoint      string
	UseSSL        bool
	BucketName    string
	User          string
	Password      string
	Token         string
	WatchFolder   string
	ExcludeFolder string
	GraceMilliSec int
	MinioClient   *minio.Client
	Ctx           context.Context
}

func NewS3

func NewS3() *S3

func (S3) GetFileSet

func (s3 S3) GetFileSet(minioclient *minio.Client) map[string]int64

func (S3) GetFilteredFileSet

func (s3 S3) GetFilteredFileSet(filter string, minioClient *minio.Client) map[string]int64

func (S3) ListFiles

func (s3 S3) ListFiles(minioClient *minio.Client) map[string]api.FileEntity

func (*S3) Watch

func (s3 *S3) Watch(eventHandlers []api.IOEvent) bool