config

Documentation

Index

• Constants
• Variables
• func AppName() string
• func ValidateClientConfig(conf *ClientConfig) error
• func ValidateServerConfig(_ *ServerConfig) error
• type Auth
• type ClientConfig
  • func LoadClientConfig(filePath string) (*ClientConfig, error)
  • func (c *ClientConfig) GetAllNamespaceNames() []string
  • func (c *ClientConfig) GetNamespaceByName(name string) (*Namespace, error)
  • func (c *ClientConfig) GetSelectedNamespaces(namespaceNames ...string) ([]*Namespace, error)
  • func (c *ClientConfig) ValidateNamespaceNames(namespaceNames ...string) error
• type Config
• type DBConfig
• type Datastore
• type Job
• type LogConfig
• type LogLevel
  • func (l LogLevel) String() string
• type Namespace
• type Optimus
  • func (o *Optimus) GetNamespaceByName(name string) (*Namespace, error)
  • func (o *Optimus) GetVersion() string
• type PluginConfig
• type Project
• type Publisher
• type PublisherKafkaConfig
• type ReplayConfig
• type ResourceManager
• type ResourceManagerConfigOptimus
• type SchedulerConfig
• type Serve
• type ServerConfig
  • func LoadServerConfig(filePath string) (*ServerConfig, error)
• type TelemetryConfig
• type Version
  • func (v Version) String() string

Constants

const (
	ServerName = "optimus"
	ClientName = "optimus-cli"
)

const (
	DefaultFilename       = "optimus.yaml"
	DefaultConfigFilename = "config.yaml" // default file name for server config
	DefaultFileExtension  = "yaml"
	DefaultEnvPrefix      = "OPTIMUS"
	EmptyPath             = ""
)

Variables

var (
	// overridden by the build system
	BuildVersion = "dev"
	BuildCommit  = ""
	BuildDate    = ""
)

var FS = afero.NewReadOnlyFs(afero.NewOsFs())

Functions

func AppName

func AppName() string

AppName returns the name used as identifier in telemetry

func ValidateClientConfig

func ValidateClientConfig(conf *ClientConfig) error

func ValidateServerConfig

func ValidateServerConfig(_ *ServerConfig) error

Types

type Auth

type Auth struct {
	ClientID     string `mapstructure:"client_id"`
	ClientSecret string `mapstructure:"client_secret"`
}

type ClientConfig

type ClientConfig struct {
	Version    Version      `mapstructure:"version"`
	Log        LogConfig    `mapstructure:"log"`
	Host       string       `mapstructure:"host"` // optimus server host
	Project    Project      `mapstructure:"project"`
	Namespaces []*Namespace `mapstructure:"namespaces"`
	Auth       Auth         `mapstructure:"auth"`
	// contains filtered or unexported fields
}

func LoadClientConfig

func LoadClientConfig(filePath string) (*ClientConfig, error)

LoadClientConfig load the project specific config from these locations: 1. filepath. ./optimus <client_command> -c "path/to/config/optimus.yaml" 2. current dir. Optimus will look at current directory if there's optimus.yaml there, use it

func (*ClientConfig) GetAllNamespaceNames

func (c *ClientConfig) GetAllNamespaceNames() []string

func (*ClientConfig) GetNamespaceByName

func (c *ClientConfig) GetNamespaceByName(name string) (*Namespace, error)

func (*ClientConfig) GetSelectedNamespaces

func (c *ClientConfig) GetSelectedNamespaces(namespaceNames ...string) ([]*Namespace, error)

func (*ClientConfig) ValidateNamespaceNames

func (c *ClientConfig) ValidateNamespaceNames(namespaceNames ...string) error

type Config

type Config interface{}

Config is just an alias for interface{}

type DBConfig

type DBConfig struct {
	DSN               string `mapstructure:"dsn"`                              // data source name e.g.: postgres://user:password@host:123/database?sslmode=disable
	MinOpenConnection int    `mapstructure:"min_open_connection" default:"5"`  // minimum open DB connections
	MaxOpenConnection int    `mapstructure:"max_open_connection" default:"20"` // maximum allowed open DB connections
}

type Datastore

type Datastore struct {
	Type   string            `mapstructure:"type"`   // type could be bigquery/postgres/gcs
	Path   string            `mapstructure:"path"`   // directory to find specifications
	Backup map[string]string `mapstructure:"backup"` // backup configuration
}

type Job

type Job struct {
	Path string `mapstructure:"path"` // directory to find specifications
}

type LogConfig

type LogConfig struct {
	Level  LogLevel `mapstructure:"level" default:"INFO"` // log level - debug, info, warning, error, fatal
	Format string   `mapstructure:"format"`               // format strategy - plain, json
}

type LogLevel

type LogLevel string

const (
	LogLevelDebug   LogLevel = "DEBUG"
	LogLevelInfo    LogLevel = "INFO"
	LogLevelWarning LogLevel = "WARNING"
	LogLevelError   LogLevel = "ERROR"
	LogLevelFatal   LogLevel = "FATAL"
)

func (LogLevel) String

func (l LogLevel) String() string

type Namespace

type Namespace struct {
	Name      string            `mapstructure:"name"`
	Config    map[string]string `mapstructure:"config"`
	Job       Job               `mapstructure:"job"`
	Datastore []Datastore       `mapstructure:"datastore"`
}

type Optimus

type Optimus struct {
	// configuration version
	Version int `mapstructure:"version"`
	// optimus server host
	Host string `mapstructure:"host"`

	Project    Project      `mapstructure:"project"`
	Namespaces []*Namespace `mapstructure:"namespaces"`

	Server    Serve           `mapstructure:"serve"`
	Log       LogConfig       `mapstructure:"log"`
	Scheduler SchedulerConfig `mapstructure:"scheduler"`
	Telemetry TelemetryConfig `mapstructure:"telemetry"`
	// contains filtered or unexported fields
}

func (*Optimus) GetNamespaceByName

func (o *Optimus) GetNamespaceByName(name string) (*Namespace, error)

func (*Optimus) GetVersion

func (o *Optimus) GetVersion() string

type PluginConfig

type PluginConfig struct {
	Artifacts []string `mapstructure:"artifacts"`
}

type Project

type Project struct {
	Name   string            `mapstructure:"name"`
	Config map[string]string `mapstructure:"config"`
}

type Publisher

type Publisher struct {
	Type   string      `mapstructure:"type" default:"kafka"`
	Buffer int         `mapstructure:"buffer"`
	Config interface{} `mapstructure:"config"`
}

type PublisherKafkaConfig

type PublisherKafkaConfig struct {
	Topic               string   `mapstructure:"topic"`
	BatchIntervalSecond int      `mapstructure:"batch_interval_second"`
	BrokerURLs          []string `mapstructure:"broker_urls"`
}

type ReplayConfig

type ReplayConfig struct {
	ReplayTimeout time.Duration `mapstructure:"replay_timeout" default:"3h"`
}

TODO: add worker interval

type ResourceManager

type ResourceManager struct {
	Name        string      `mapstructure:"name"`
	Type        string      `mapstructure:"type"`
	Description string      `mapstructure:"description"`
	Config      interface{} `mapstructure:"config"`
}

type ResourceManagerConfigOptimus

type ResourceManagerConfigOptimus struct {
	Host    string            `mapstructure:"host"`
	Headers map[string]string `mapstructure:"headers"`
}

type SchedulerConfig

type SchedulerConfig struct {
	Name string `mapstructure:"name" default:"airflow"`
}

type Serve

type Serve struct {
	Port        int      `mapstructure:"port" default:"9100"` // port to listen on
	IngressHost string   `mapstructure:"ingress_host"`        // service ingress host for jobs to communicate back to optimus
	AppKey      string   `mapstructure:"app_key"`             // random 32 character hash used for encrypting secrets
	DB          DBConfig `mapstructure:"db"`
}

type ServerConfig

type ServerConfig struct {
	Version          Version           `mapstructure:"version"`
	Log              LogConfig         `mapstructure:"log"`
	Serve            Serve             `mapstructure:"serve"`
	Scheduler        SchedulerConfig   `mapstructure:"scheduler"`
	Telemetry        TelemetryConfig   `mapstructure:"telemetry"`
	ResourceManagers []ResourceManager `mapstructure:"resource_managers"`
	Plugin           PluginConfig      `mapstructure:"plugin"`
	Replay           ReplayConfig      `mapstructure:"replay"`
	Publisher        *Publisher        `mapstructure:"publisher"`
}

func LoadServerConfig

func LoadServerConfig(filePath string) (*ServerConfig, error)

LoadServerConfig load the server specific config from these locations: 1. filepath. ./optimus <server_command> -c "path/to/config.yaml" 2. env var. eg. OPTIMUS_SERVE_PORT, etc 3. executable binary location

type TelemetryConfig

type TelemetryConfig struct {
	ProfileAddr string `mapstructure:"profile_addr"`
	JaegerAddr  string `mapstructure:"jaeger_addr"`
}

type Version

type Version int

Version implement fmt.Stringer

const DefaultVersion Version = 1

func (Version) String

func (v Version) String() string