config

Documentation

Index

• func Asset(name string) ([]byte, error)
• func AssetDir(name string) ([]string, error)
• func AssetInfo(name string) (os.FileInfo, error)
• func AssetNames() []string
• func MustAsset(name string) []byte
• func Parse(raw []byte, conf *Config) error
• func ParseFile(relpath string, conf *Config) error
• func RestoreAsset(dir, name string) error
• func RestoreAssets(dir, name string) error
• type AWSBatch
• type AWSConfig
• type Config
  • func DefaultConfig() Config
  • func EnsureServerProperties(conf Config) Config
  • func (c Config) ToYaml() []byte
  • func (c Config) ToYamlFile(p string)
  • func (c Config) ToYamlTempFile(name string) (string, func())
• type DynamoDB
• type Elastic
• type GSStorage
  • func (g GSStorage) Valid() bool
• type Kafka
• type LocalStorage
  • func (l LocalStorage) Valid() bool
• type MongoDB
• type Node
• type RPC
• type S3Storage
  • func (s S3Storage) Valid() bool
• type Scheduler
• type Server
  • func (c Server) HTTPAddress() string
  • func (c *Server) RPCAddress() string
• type StorageConfig
• type SwiftStorage
  • func (s SwiftStorage) Valid() bool
• type Worker

Functions

func Asset

func Asset(name string) ([]byte, error)

Asset loads and returns the asset for the given name. It returns an error if the asset could not be found or could not be loaded.

func AssetDir

func AssetDir(name string) ([]string, error)

AssetDir returns the file names below a certain directory embedded in the file by go-bindata. For example if you run go-bindata on data/... and data contains the following hierarchy:

data/
  foo.txt
  img/
    a.png
    b.png

then AssetDir("data") would return []string{"foo.txt", "img"} AssetDir("data/img") would return []string{"a.png", "b.png"} AssetDir("foo.txt") and AssetDir("notexist") would return an error AssetDir("") will return []string{"data"}.

func AssetInfo

func AssetInfo(name string) (os.FileInfo, error)

AssetInfo loads and returns the asset info for the given name. It returns an error if the asset could not be found or could not be loaded.

func AssetNames

func AssetNames() []string

AssetNames returns the names of the assets.

func MustAsset

func MustAsset(name string) []byte

MustAsset is like Asset but panics when Asset would return an error. It simplifies safe initialization of global variables.

func Parse

func Parse(raw []byte, conf *Config) error

Parse parses a YAML doc into the given Config instance.

func ParseFile

func ParseFile(relpath string, conf *Config) error

ParseFile parses a Funnel config file, which is formatted in YAML, and returns a Config struct.

func RestoreAsset

func RestoreAsset(dir, name string) error

RestoreAsset restores an asset under the given directory

func RestoreAssets

func RestoreAssets(dir, name string) error

RestoreAssets restores an asset under the given directory recursively

Types

type AWSBatch

type AWSBatch struct {
	// JobDefinition can be either a name or the Amazon Resource Name (ARN).
	JobDefinition string
	// JobQueue can be either a name or the Amazon Resource Name (ARN).
	JobQueue string
	AWS      AWSConfig
}

AWSBatch describes the configuration for the AWS Batch compute backend.

type AWSConfig

type AWSConfig struct {
	// An optional endpoint URL (hostname only or fully qualified URI)
	// that overrides the default generated endpoint for a client.
	Endpoint string
	// The region to send requests to. This parameter is required and must
	// be configured on a per-client basis for all AWS services in funnel with the
	// exception of S3Storage.
	Region string
	// The maximum number of times that a request will be retried for failures.
	// By default defers the max retry setting to the service
	// specific configuration. Set to a value > 0 to override.
	MaxRetries int
	// If both the key and secret are empty AWS credentials will be read from
	// the environment.
	Key    string
	Secret string
}

AWSConfig describes the configuration for creating AWS Session instances

type Config

type Config struct {
	Server Server
	// the active compute backend
	Backend  string
	Backends struct {
		Local    struct{}
		HTCondor struct {
			Template string
		}
		SLURM struct {
			Template string
		}
		PBS struct {
			Template string
		}
		GridEngine struct {
			Template string
		}
		OpenStack struct {
			KeyPair    string
			ConfigPath string
			Server     os_servers.CreateOpts
		}
		GCE struct {
			AccountFile string
			Project     string
			Zone        string
			Weights     struct {
				PreferQuickStartup float32
			}
			CacheTTL time.Duration
		}
		Batch AWSBatch
	}
	Scheduler Scheduler
	Worker    Worker
}

Config describes configuration for Funnel.

func DefaultConfig

func DefaultConfig() Config

DefaultConfig returns configuration with simple defaults.

func EnsureServerProperties

func EnsureServerProperties(conf Config) Config

EnsureServerProperties ensures that the server address and server password is consistent between the worker, node, and server.

func (Config) ToYaml

func (c Config) ToYaml() []byte

ToYaml formats the configuration into YAML and returns the bytes.

func (Config) ToYamlFile

func (c Config) ToYamlFile(p string)

ToYamlFile writes the configuration to a YAML file.

func (Config) ToYamlTempFile

func (c Config) ToYamlTempFile(name string) (string, func())

ToYamlTempFile writes the configuration to a YAML temp. file.

type DynamoDB

type DynamoDB struct {
	AWS           AWSConfig
	TableBasename string
}

DynamoDB describes the configuration for Amazon DynamoDB backed processes such as the event writer and server.

type Elastic

type Elastic struct {
	IndexPrefix string
	URL         string
}

Elastic configures access to an Elasticsearch database.

type GSStorage

type GSStorage struct {
	AccountFile string
	FromEnv     bool
}

GSStorage describes configuration for the Google Cloud storage backend.

func (GSStorage) Valid

func (g GSStorage) Valid() bool

Valid validates the GSStorage configuration.

type Kafka

type Kafka struct {
	Servers []string
	Topic   string
}

Kafka configure access to a Kafka topic for task event reading/writing.

type LocalStorage

type LocalStorage struct {
	AllowedDirs []string
}

LocalStorage describes the directories Funnel can read from and write to

func (LocalStorage) Valid

func (l LocalStorage) Valid() bool

Valid validates the LocalStorage configuration

type MongoDB

type MongoDB struct {
	// Addrs holds the addresses for the seed servers.
	Addrs []string
	// Database is the database name used within MongoDB to store funnel data.
	Database string
	Username string
	Password string
}

MongoDB configures access to an MongoDB database.

type Node

type Node struct {
	ID      string
	WorkDir string
	// A Node will automatically try to detect what resources are available to it.
	// Defining Resources in the Node configuration overrides this behavior.
	Resources struct {
		Cpus   uint32
		RamGb  float64 // nolint
		DiskGb float64
	}
	// If the node has been idle for longer than the timeout, it will shut down.
	// -1 means there is no timeout. 0 means timeout immediately after the first task.
	Timeout time.Duration
	// How often the node sends update requests to the server.
	UpdateRate time.Duration
	// Timeout duration for PutNode() gRPC calls
	UpdateTimeout time.Duration
	Metadata      map[string]string
	// RPC address of the Funnel server
	ServerAddress string
	// Password for basic auth. with the server APIs.
	ServerPassword string
	Logger         logger.Config
}

Node contains the configuration for a node. Nodes track available resources for funnel's basic scheduler.

type RPC

type RPC struct {
	// RPC address of the Funnel server
	ServerAddress string
	// Password for basic auth. with the server APIs.
	ServerPassword string
	// Timeout duration for gRPC calls
	Timeout time.Duration
}

RPC configures access to the Funnel RPC server.

type S3Storage

type S3Storage struct {
	Disabled bool
	AWS      AWSConfig
}

S3Storage describes the directories Funnel can read from and write to

func (S3Storage) Valid

func (s S3Storage) Valid() bool

Valid validates the LocalStorage configuration

type Scheduler

type Scheduler struct {
	// How often to run a scheduler iteration.
	ScheduleRate time.Duration
	// How many tasks to schedule in one iteration.
	ScheduleChunk int
	// How long to wait for a node ping before marking it as dead
	NodePingTimeout time.Duration
	// How long to wait for node initialization before marking it dead
	NodeInitTimeout time.Duration
	// How long to wait before deleting a dead node from the DB.
	NodeDeadTimeout time.Duration
	// Node configuration
	Node Node
	// Logger configuration
	Logger logger.Config
}

Scheduler contains funnel's basic scheduler configuration.

type Server

type Server struct {
	ServiceName string
	HostName    string
	HTTPPort    string
	RPCPort     string
	Password    string
	Database    string
	Databases   struct {
		BoltDB struct {
			Path string
		}
		DynamoDB DynamoDB
		Elastic  Elastic
		MongoDB  MongoDB
	}
	DisableHTTPCache bool
	Logger           logger.Config
}

Server describes configuration for the server.

func (Server) HTTPAddress

func (c Server) HTTPAddress() string

HTTPAddress returns the HTTP address based on HostName and HTTPPort

func (*Server) RPCAddress

func (c *Server) RPCAddress() string

RPCAddress returns the RPC address based on HostName and RPCPort

type StorageConfig

type StorageConfig struct {
	Local LocalStorage
	S3    S3Storage
	GS    []GSStorage
	Swift SwiftStorage
}

StorageConfig describes configuration for all storage types

type SwiftStorage

type SwiftStorage struct {
	Disabled   bool
	UserName   string
	Password   string
	AuthURL    string
	TenantName string
	TenantID   string
	RegionName string
}

SwiftStorage configures the OpenStack Swift object storage backend.

func (SwiftStorage) Valid

func (s SwiftStorage) Valid() bool

Valid validates the SwiftStorage configuration.

type Worker

type Worker struct {
	// Directory to write task files to
	WorkDir string
	// How often the worker sends task log updates
	UpdateRate time.Duration
	// Max bytes to store in-memory between updates
	BufferSize  int64
	Storage     StorageConfig
	Logger      logger.Config
	TaskReader  string
	TaskReaders struct {
		RPC      RPC
		DynamoDB DynamoDB
		Elastic  Elastic
		MongoDB  MongoDB
	}
	ActiveEventWriters []string
	EventWriters       struct {
		RPC      RPC
		DynamoDB DynamoDB
		Elastic  Elastic
		MongoDB  MongoDB
		Kafka    Kafka
	}
}

Worker contains worker configuration.