assembly

Documentation

Index

• type BigQueryConfig
  • func (b BigQueryConfig) Close() error
  • func (b *BigQueryConfig) InitClient(ctx context.Context, projectId string) (err error)
• type BigTableConfig
  • func (b BigTableConfig) Close() error
  • func (b *BigTableConfig) InitClients(ctx context.Context, projectId string) (err error)
• type Config
  • func (c Config) Close() error
• type FirestoreConfig
  • func (f *FirestoreConfig) InitClient(ctx context.Context, projectId string) (err error)
• type KafkaConfig
  • func (k KafkaConfig) DefaultBootstrapServers(provider string) string
• type PubsubConfig
  • func (p *PubsubConfig) InitClient(ctx context.Context, projectId string) (err error)
• type StreamEntityFactory
  • func NewStreamEntityFactory(config Config) *StreamEntityFactory
  • func (s *StreamEntityFactory) CreateExtractor(ctx context.Context, etlSpec igeist.Spec, instanceId string) (igeist.Extractor, error)
  • func (s *StreamEntityFactory) CreateLoader(ctx context.Context, etlSpec igeist.Spec, instanceId string) (igeist.Loader, error)
  • func (s *StreamEntityFactory) CreateSinkExtractor(ctx context.Context, etlSpec igeist.Spec, instanceId string) (igeist.Extractor, error)
  • func (s *StreamEntityFactory) CreateTransformer(ctx context.Context, etlSpec igeist.Spec) (igeist.Transformer, error)
  • func (s *StreamEntityFactory) SetAdminLoader(loader igeist.Loader)

Types

type BigQueryConfig

type BigQueryConfig struct {
	Enabled bool
	Client  *bigquery.Client
}

func (BigQueryConfig) Close

func (b BigQueryConfig) Close() error

func (*BigQueryConfig) InitClient

func (b *BigQueryConfig) InitClient(ctx context.Context, projectId string) (err error)

type BigTableConfig

type BigTableConfig struct {
	Enabled     bool
	InstanceId  string
	Client      *bigtable.Client
	AdminClient *bigtable.AdminClient
}

func (BigTableConfig) Close

func (b BigTableConfig) Close() error

func (*BigTableConfig) InitClients

func (b *BigTableConfig) InitClients(ctx context.Context, projectId string) (err error)

type Config

type Config struct {
	Env       model.Environment
	Kafka     KafkaConfig
	BigTable  BigTableConfig
	BigQuery  BigQueryConfig
	Firestore FirestoreConfig
	Pubsub    PubsubConfig
}

func (Config) Close

func (c Config) Close() error

type FirestoreConfig

type FirestoreConfig struct {
	Enabled bool
	Client  *datastore.Client
}

func (*FirestoreConfig) InitClient

func (f *FirestoreConfig) InitClient(ctx context.Context, projectId string) (err error)

type KafkaConfig

type KafkaConfig struct {
	Enabled                  bool
	BootstrapServers         string // Default servers, can be overriden in GEIST specs
	ConfluentBootstrapServer string // Default server, can be overriden in GEIST specs
	ConfluentApiKey          string
	ConfluentApiSecret       string `json:"-"`
	PollTimeoutMs            int    // Default poll timeout, can be overridden in GEIST specs
	QueuedMaxMessagesKb      int    // Events consumed and processed before commit
}

func (KafkaConfig) DefaultBootstrapServers

func (k KafkaConfig) DefaultBootstrapServers(provider string) string

Convenience function for easy default config

type PubsubConfig

type PubsubConfig struct {
	Enabled                bool
	Client                 *pubsub.Client
	MaxOutstandingMessages int
	MaxOutstandingBytes    int
}

func (*PubsubConfig) InitClient

func (p *PubsubConfig) InitClient(ctx context.Context, projectId string) (err error)

type StreamEntityFactory

type StreamEntityFactory struct {
	// contains filtered or unexported fields
}

StreamEntityFactory creates stream entities based on Stream Spec config and is a singleton, created by the Service, and operated by the StreamBuilder (also a singleton), which is given to the Supervisor during creation.

func NewStreamEntityFactory

func NewStreamEntityFactory(config Config) *StreamEntityFactory

func (*StreamEntityFactory) CreateExtractor

func (s *StreamEntityFactory) CreateExtractor(ctx context.Context, etlSpec igeist.Spec, instanceId string) (igeist.Extractor, error)

func (*StreamEntityFactory) CreateLoader

func (s *StreamEntityFactory) CreateLoader(ctx context.Context, etlSpec igeist.Spec, instanceId string) (igeist.Loader, error)

func (*StreamEntityFactory) CreateSinkExtractor

func (s *StreamEntityFactory) CreateSinkExtractor(ctx context.Context, etlSpec igeist.Spec, instanceId string) (igeist.Extractor, error)

func (*StreamEntityFactory) CreateTransformer

func (s *StreamEntityFactory) CreateTransformer(ctx context.Context, etlSpec igeist.Spec) (igeist.Transformer, error)

func (*StreamEntityFactory) SetAdminLoader

func (s *StreamEntityFactory) SetAdminLoader(loader igeist.Loader)