config

Documentation

Index

• Constants
• func GetPodCPULimit() string
• func GetPodMemoryLimit() string
• type Config
  • func NewConfig(listenIP string, listenPort int, p string) (*Config, error)
  • func NewConfigFromJSON(cString string) (c *Config, fnerr error)
  • func NewConfigFromPath(p string) (*Config, error)
  • func NewDefaultConfig(masterIP string, masterPort int32) (*Config, error)
  • func SaveConfig(c *Config, p string) (*Config, error)
  • func (c *Config) Get(vname string) (string, error)
  • func (c *Config) GetBool(vname string) bool
  • func (c *Config) GetDomain() string
  • func (c *Config) GetFlowNS() string
  • func (c *Config) GetInt32(vname string) (int32, error)
  • func (c *Config) GetJobs() *JobConfig
  • func (c *Config) GetListenAddr() string
  • func (c *Config) GetNS() string
  • func (c *Config) GetNbScheduler() *NbSchedulerConfig
  • func (c *Config) GetNotebooks() *NotebookConfig
• type DBConfig
• type DbTarget
• type GcsConfig
• type JobConfig
• type KubeConfig
• type NbSchedulerConfig
• type NotebookConfig
  • func (nbc *NotebookConfig) GetBackgroundCmd() string
  • func (nbc *NotebookConfig) GetCommand() string
• type ObjStorageConfig
• type PodConfig
• type S3Config
• type StorageTarget

Constants

const (
	// Bolt : Database Option
	Bolt DbTarget = "BOLT"

	// Postgres : Database option
	Postgres DbTarget = "POSTGRES"

	// Badger : DB Option
	Badger DbTarget = "BADGER"

	// DefaultDBEventBuffer : Change Listern Buffer length
	DefaultDBEventBuffer = 500

	// DefaultDataDirPath : Used by Bolt/Badger as data file path
	DefaultDataDirPath = "/data/badger"

	// DefaultDBBucket : Used by Bolt/Badger to name the bucket
	DefaultDBBucket = "hflow_master"
)

const (
	// GCS : store objects / log on google cloud
	GCS StorageTarget = "GCS"

	// S3 : store objects / log on AWS
	S3 StorageTarget = "S3"

	// DefaultStorageBucket : Default bucket for object storage
	DefaultStorageBucket = "hyperflow"

	// DefaultStorageRegion : Default region used for object storage
	DefaultStorageRegion = "us-west-2"

	// DefaultStorageTarget : Default Backend for object storage
	DefaultStorageTarget = GCS

	// DefaultStorageBaseDir : Default base directory for backend. Used by Object storage
	DefaultStorageBaseDir = "hyperflow"
)

const (

	// ConfigVarsEnvName : config file vars
	ConfigVarsEnvName = "HF_SERVER_CONFIG_VARS"

	// ConfigPathEnvName : path to config file
	ConfigPathEnvName = "HF_SERVER_CONFIG_PATH"

	// Safemode param
	Safemode = "Safemode"

	// DefaultMasterPort : Default port for master server
	DefaultMasterPort = int32(3200)

	// ConfigFilePerm : Permissions assigned on creation a new config file
	ConfigFilePerm = 0644
)

const (
	// DefaultNamespace  : Default K8S Namespace
	DefaultNamespace = "hyperml"

	// DefaultPodCPULimit : Default POD CPU Limit
	DefaultPodCPULimit = "0.2"

	// DefaultPodMemoryLimit : Default POD Memory Limit
	DefaultPodMemoryLimit = "100M"

	// DefaultDomain : domain used to launch notebooks/labs
	DefaultDomain = "hyperml.com"
)

const (
	// NbDefaultCommand : Default command for launching notebooks
	NbDefaultCommand = "jupyter notebook --no-browser --port={port} --ip={ip}  --NotebookApp.port_retries=0 --NotebookApp.disable_check_xsrf=True" // "--NotebookApp.token=abc"

	// NbDefaultBackgroundCmd : Default command used for background notebooks
	NbDefaultBackgroundCmd = "papermill "

	// NbDefaultSweepInterval : Default Notebook scheduler sweep interval
	NbDefaultSweepInterval = 1 // minutes

	// NbDefaultSaveInterval : Default Notebook scheduler save interval
	NbDefaultSaveInterval = 60 //seconds

	// JobDefaultBackoff : Job Default bckoff
	JobDefaultBackoff = 1

	// JobDefaultDeadline : Job Default active deadline
	JobDefaultDeadline = 54000

	// DefaultNotebookPort : Default port used for hosting notebook in the container
	DefaultNotebookPort = "8888"

	// DefaultNotebookIP : Default IP used by notebook container
	DefaultNotebookIP = "0.0.0.0"

	// DefaultNotebookBasePath: Used as base path by Notebook container
	DefaultNotebookBasePath = "/"
)

Constants associated with notebook processor

Functions

func GetPodCPULimit

func GetPodCPULimit() string

GetPodCPULimit : Returns default Pod Memory Limit

func GetPodMemoryLimit

func GetPodMemoryLimit() string

GetPodMemoryLimit : Returns default Pod Memory Limit

Types

type Config

type Config struct {
	Namespace          string
	FlowNamespace      string
	PublicInterface    string
	MasterIP           string
	MasterPort         int32
	MasterExternalPort int32
	NoSSL              bool
	NoAuth             bool
	DisableFlow        bool

	DB          *DBConfig
	K8          *KubeConfig
	ObjStorage  *ObjStorageConfig
	Notebooks   *NotebookConfig
	NbScheduler *NbSchedulerConfig

	Jobs *JobConfig
	Pods *PodConfig

	Domain   string
	LogLevel int
	LogPath  string
	Safemode bool
}

Config : holds entire configuration of hflow server

func NewConfig

func NewConfig(listenIP string, listenPort int, p string) (*Config, error)

NewConfig : Used to fire up new configuration object

func NewConfigFromJSON

func NewConfigFromJSON(cString string) (c *Config, fnerr error)

NewConfigFromJSON : Sets config file with json string input

func NewConfigFromPath

func NewConfigFromPath(p string) (*Config, error)

NewConfigFromPath : Used when a config path is set and file exists

func NewDefaultConfig

func NewDefaultConfig(masterIP string, masterPort int32) (*Config, error)

NewDefaultConfig : Default Config used when stored config is not found

func SaveConfig

func SaveConfig(c *Config, p string) (*Config, error)

SaveConfig : Saved config file to a give path (default $HOME/.hflow)

func (*Config) Get

func (c *Config) Get(vname string) (string, error)

Get : Get string config vars

func (*Config) GetBool

func (c *Config) GetBool(vname string) bool

GetBool : Get boolean config vars

func (*Config) GetDomain

func (c *Config) GetDomain() string

GetDomain : Get domain used to generate URL for launch notebooks/labs

func (*Config) GetFlowNS

func (c *Config) GetFlowNS() string

GetFlowNS : Get kubernetes namespace to be used by Flow

func (*Config) GetInt32

func (c *Config) GetInt32(vname string) (int32, error)

GetInt32 : Get Int32 config vars

func (*Config) GetJobs

func (c *Config) GetJobs() *JobConfig

GetJobs : Get Job specific config from default object

func (*Config) GetListenAddr

func (c *Config) GetListenAddr() string

GetListenAddr : Get HTTP listener address for main server

func (*Config) GetNS

func (c *Config) GetNS() string

GetNS : Get namespace for kuberentes cluster

func (*Config) GetNbScheduler

func (c *Config) GetNbScheduler() *NbSchedulerConfig

GetNbScheduler : Returns Notebook Scheduler Config

func (*Config) GetNotebooks

func (c *Config) GetNotebooks() *NotebookConfig

GetNotebooks : Returns Notebook Config if set. else Default

type DBConfig

type DBConfig struct {
	Driver      DbTarget // POSTGRES, BOLT
	Name        string
	User        string
	Pass        string
	DataDirPath string
	//Change Listener Threshold
	EventBuffer int
}

DBConfig : Stores config for metadata DB

type DbTarget

type DbTarget string

DbTarget : Identifies database option (postgres, badger, bolt etc)

type GcsConfig

type GcsConfig struct {
	CredsPath string
	Bucket    string
	Creds     []byte
}

GcsConfig : Stored Object Storage GCS backend config

type JobConfig

type JobConfig struct {
	DeadlineSeconds int
	BackoffLimit    int
	TTL             int32
}

JobConfig : Job specific config

type KubeConfig

type KubeConfig struct {
	Namespace string
	Path      string
	InCluster bool
}

KubeConfig : Stores K8S connection info

type NbSchedulerConfig

type NbSchedulerConfig struct {
	SweepInterval int
	SaveInterval  int
	Concurrency   int
}

NbSchedulerConfig : Configuration Parameters for Notebook Scheduler

type NotebookConfig

type NotebookConfig struct {
	Command       string
	Port          string
	IP            string
	BasePath      string
	BackgroundCmd string
}

NotebookConfig : Configuration Parameters for Notebook instance

func (*NotebookConfig) GetBackgroundCmd

func (nbc *NotebookConfig) GetBackgroundCmd() string

GetBackgroundCmd : Returns default command to launch background notebook

func (*NotebookConfig) GetCommand

func (nbc *NotebookConfig) GetCommand() string

GetCommand : Returns Default command to launch notebook

type ObjStorageConfig

type ObjStorageConfig struct {
	StorageTarget StorageTarget
	BaseDir       string
	S3            *S3Config
	Gcs           *GcsConfig
}

ObjStorageConfig : Stored Object Storage backend config

type PodConfig

type PodConfig struct {
	TTL int32
}

PodConfig : Job specific config

type S3Config

type S3Config struct {
	CredPath     string
	AccessKey    string
	SecretKey    string
	SessionToken string
	Bucket       string
	Region       string
	Creds        string
}

S3Config : Stored Object Storage S3 backend config

type StorageTarget

type StorageTarget string

StorageTarget : Identifies target backend for object storage