config

Documentation

Overview

Package config contains Funnel configuration structures and defaults.

Index

• func Examples() map[string]string
• func Parse(raw []byte, conf *Config) error
• func ParseFile(relpath string, conf *Config) error
• func ToYaml(c Config) ([]byte, error)
• func ToYamlFile(c Config, path string) error
• type AWSBatch
• type AWSConfig
• type AmazonS3Storage
  • func (s AmazonS3Storage) Valid() bool
• type Badger
• type BasicCredential
• type BoltDB
• type Config
  • func DefaultConfig() Config
• type Datastore
• type Duration
  • func (d Duration) MarshalText() (text []byte, err error)
  • func (d *Duration) Set(raw string) error
  • func (d *Duration) String() string
  • func (d *Duration) Type() string
  • func (d *Duration) UnmarshalText(text []byte) error
• type DynamoDB
• type Elastic
• type FTPStorage
  • func (h FTPStorage) Valid() bool
• type GenericS3Storage
  • func (s GenericS3Storage) Valid() bool
• type GoogleCloudStorage
  • func (g GoogleCloudStorage) Valid() bool
• type HPCBackend
• type HTTPStorage
  • func (h HTTPStorage) Valid() bool
• type Kafka
• type Kubernetes
• type LocalStorage
  • func (l LocalStorage) Valid() bool
• type MongoDB
• type Node
• type PubSub
• type RPCClient
• type Scheduler
• type Server
  • func (c Server) HTTPAddress() string
  • func (c *Server) RPCAddress() string
• type SwiftStorage
  • func (s SwiftStorage) Valid() bool
• type Worker

Functions

func Examples

func Examples() map[string]string

Examples returns a set of example configurations strings.

func Parse

func Parse(raw []byte, conf *Config) error

Parse parses a YAML doc into the given Config instance.

func ParseFile

func ParseFile(relpath string, conf *Config) error

ParseFile parses a Funnel config file, which is formatted in YAML, and returns a Config struct.

func ToYaml

func ToYaml(c Config) ([]byte, error)

ToYaml formats the configuration into YAML and returns the bytes.

func ToYamlFile

func ToYamlFile(c Config, path string) error

ToYamlFile writes the configuration to a YAML file.

Types

type AWSBatch

type AWSBatch struct {
	// JobDefinition can be either a name or the Amazon Resource Name (ARN).
	JobDefinition string
	// JobQueue can be either a name or the Amazon Resource Name (ARN).
	JobQueue string
	// Turn off task state reconciler. When enabled, Funnel communicates with AWS Batch
	// to find tasks that never started and updates task state accordingly.
	DisableReconciler bool
	// ReconcileRate is how often the compute backend compares states in Funnel's backend
	// to those reported by AWS Batch
	ReconcileRate Duration
	AWSConfig
}

AWSBatch describes the configuration for the AWS Batch compute backend.

type AWSConfig

type AWSConfig struct {
	// An optional endpoint URL (hostname only or fully qualified URI)
	// that overrides the default generated endpoint for a client.
	Endpoint string
	// The region to send requests to. This parameter is required and must
	// be configured on a per-client basis for all AWS services in funnel with the
	// exception of S3Storage.
	Region string
	// The maximum number of times that a request will be retried for failures.
	// By default defers the max retry setting to the service
	// specific configuration. Set to a value > 0 to override.
	MaxRetries int
	// If both the key and secret are empty AWS credentials will be read from
	// the environment.
	Key    string
	Secret string
	// Don't automatically read AWS credentials from the environment.
	DisableAutoCredentialLoad bool
}

AWSConfig describes the configuration for creating AWS Session instances

type AmazonS3Storage

type AmazonS3Storage struct {
	Disabled bool
	SSE      struct {
		CustomerKeyFile string
		KMSKey          string
	}
	AWSConfig
}

AmazonS3Storage describes the configuration for the Amazon S3 storage backend.

func (AmazonS3Storage) Valid

func (s AmazonS3Storage) Valid() bool

Valid validates the AmazonS3Storage configuration

type Badger

type Badger struct {
	// Path to database directory.
	Path string
}

Badger describes configuration for the Badger embedded database.

type BasicCredential

type BasicCredential struct {
	User     string
	Password string
}

BasicCredential describes a username and password for use with Funnel's basic auth.

type BoltDB

type BoltDB struct {
	Path string
}

BoltDB describes the configuration for the BoltDB embedded database.

type Config

type Config struct {
	// component selectors
	EventWriters []string
	Database     string
	Compute      string
	// funnel components
	Server    Server
	RPCClient RPCClient
	Scheduler Scheduler
	Node      Node
	Worker    Worker
	Logger    logger.Config
	// databases / event handlers
	BoltDB    BoltDB
	Badger    Badger
	DynamoDB  DynamoDB
	Elastic   Elastic
	MongoDB   MongoDB
	Kafka     Kafka
	PubSub    PubSub
	Datastore Datastore
	// compute
	HTCondor   HPCBackend
	Slurm      HPCBackend
	PBS        HPCBackend
	GridEngine struct {
		Template     string
		TemplateFile string
	}
	AWSBatch   AWSBatch
	Kubernetes Kubernetes
	// storage
	LocalStorage  LocalStorage
	AmazonS3      AmazonS3Storage
	GenericS3     []GenericS3Storage
	GoogleStorage GoogleCloudStorage
	Swift         SwiftStorage
	HTTPStorage   HTTPStorage
	FTPStorage    FTPStorage
}

Config describes configuration for Funnel.

func DefaultConfig

func DefaultConfig() Config

DefaultConfig returns configuration with simple defaults.

type Datastore

type Datastore struct {
	Project string
	// If no account file is provided then Funnel will try to use Google Application
	// Default Credentials to authorize and authenticate the client.
	CredentialsFile string
}

Datastore configures access to a Google Cloud Datastore database backend.

type Duration

type Duration time.Duration

Duration is a wrapper type for time.Duration which provides human-friendly text (un)marshaling. See https://github.com/golang/go/issues/16039

func (Duration) MarshalText

func (d Duration) MarshalText() (text []byte, err error)

MarshalText converts a duration to text.

func (*Duration) Set

func (d *Duration) Set(raw string) error

Set sets the duration from the given string. Implements the pflag.Value interface.

func (*Duration) String

func (d *Duration) String() string

String returns the string representation of the duration.

func (*Duration) Type

func (d *Duration) Type() string

Type returns the name of this type. Implements the pflag.Value interface.

func (*Duration) UnmarshalText

func (d *Duration) UnmarshalText(text []byte) error

UnmarshalText parses text into a duration value.

type DynamoDB

type DynamoDB struct {
	TableBasename string
	AWSConfig
}

DynamoDB describes the configuration for Amazon DynamoDB backed processes such as the event writer and server.

type Elastic

type Elastic struct {
	IndexPrefix string
	URL         string
}

Elastic configures access to an Elasticsearch database.

type FTPStorage

type FTPStorage struct {
	Disabled bool
	// Timeout duration for http GET calls
	Timeout  Duration
	User     string
	Password string
}

FTPStorage configures the http storage backend.

func (FTPStorage) Valid

func (h FTPStorage) Valid() bool

Valid validates the FTPStorage configuration.

type GenericS3Storage

type GenericS3Storage struct {
	Disabled bool
	Endpoint string
	Key      string
	Secret   string
}

GenericS3Storage describes the configuration for the Generic S3 storage backend.

func (GenericS3Storage) Valid

func (s GenericS3Storage) Valid() bool

Valid validates the S3Storage configuration

type GoogleCloudStorage

type GoogleCloudStorage struct {
	Disabled bool
	// If no account file is provided then Funnel will try to use Google Application
	// Default Credentials to authorize and authenticate the client.
	CredentialsFile string
}

GoogleCloudStorage describes configuration for the Google Cloud storage backend.

func (GoogleCloudStorage) Valid

func (g GoogleCloudStorage) Valid() bool

Valid validates the Storage configuration.

type HPCBackend

type HPCBackend struct {
	// Turn off task state reconciler. When enabled, Funnel communicates with the HPC
	// scheduler to find tasks that are stuck in a queued state or errored and
	// updates the task state accordingly.
	DisableReconciler bool
	// ReconcileRate is how often the compute backend compares states in Funnel's backend
	// to those reported by the backend
	ReconcileRate Duration
	// Template is the template to use for submissions to the HPC backend
	Template string
	// TemplateFile is the path to the template to use for submissions to the HPC backend
	TemplateFile string
}

HPCBackend describes the configuration for a HPC scheduler backend such as HTCondor or Slurm.

type HTTPStorage

type HTTPStorage struct {
	Disabled bool
	// Timeout duration for http GET calls
	Timeout Duration
}

HTTPStorage configures the http storage backend.

func (HTTPStorage) Valid

func (h HTTPStorage) Valid() bool

Valid validates the HTTPStorage configuration.

type Kafka

type Kafka struct {
	Servers []string
	Topic   string
}

Kafka configure access to a Kafka topic for task event reading/writing.

type Kubernetes

type Kubernetes struct {
	// Turn off task state reconciler. When enabled, Funnel communicates with Kuberenetes
	// to find tasks that are stuck in a queued state or errored and updates the task state
	// accordingly.
	DisableReconciler bool
	// ReconcileRate is how often the compute backend compares states in Funnel's backend
	// to those reported by the backend
	ReconcileRate Duration
	// Disable cleanup of complete/failed jobs. Cleanup is run during reconcile loop.
	DisableJobCleanup bool
	// Batch job template. See: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#job-v1-batch
	Template string
	// TemplateFile is the path to the job template.
	TemplateFile string
	// Path to the Kubernetes configuration file, otherwise assumes the Funnel server is running in a pod and
	// attempts to use https://godoc.org/k8s.io/client-go/rest#InClusterConfig to infer configuration.
	ConfigFile string
	// Namespace to spawn jobs within
	Namespace string
}

Kubernetes describes the configuration for the Kubernetes compute backend.

type LocalStorage

type LocalStorage struct {
	Disabled    bool
	AllowedDirs []string
}

LocalStorage describes the directories Funnel can read from and write to

func (LocalStorage) Valid

func (l LocalStorage) Valid() bool

Valid validates the LocalStorage configuration

type MongoDB

type MongoDB struct {
	// Addrs holds the addresses for the seed servers.
	Addrs []string
	// Database is the database name used within MongoDB to store funnel data.
	Database string
	// Timeout is the amount of time to wait for a server to respond when
	// first connecting and on follow up operations in the session. If
	// timeout is zero, the call may block forever waiting for a connection
	// to be established.
	Timeout Duration
	// Username and Password inform the credentials for the initial authentication
	// done on the database defined by the Database field.
	Username string
	Password string
}

MongoDB configures access to an MongoDB database.

type Node

type Node struct {
	ID string
	// A Node will automatically try to detect what resources are available to it.
	// Defining Resources in the Node configuration overrides this behavior.
	Resources struct {
		Cpus   uint32
		RamGb  float64 // nolint
		DiskGb float64
	}
	// If the node has been idle for longer than the timeout, it will shut down.
	// -1 means there is no timeout. 0 means timeout immediately after the first task.
	Timeout Duration
	// How often the node sends update requests to the server.
	UpdateRate Duration
	Metadata   map[string]string
}

Node contains the configuration for a node. Nodes track available resources for funnel's basic scheduler.

type PubSub

type PubSub struct {
	Topic   string
	Project string
	// If no account file is provided then Funnel will try to use Google Application
	// Default Credentials to authorize and authenticate the client.
	CredentialsFile string
}

PubSub configures access to Google Cloud Pub/Sub for task event reading/writing.

type RPCClient

type RPCClient struct {
	BasicCredential
	ServerAddress string
	// The timeout to use for making RPC client connections in nanoseconds
	// This timeout is Only enforced when used in conjunction with the
	// grpc.WithBlock dial option.
	Timeout Duration
	// The maximum number of times that a request will be retried for failures.
	// Time between retries follows an exponential backoff starting at 5 seconds
	// up to 1 minute
	MaxRetries uint
}

RPCClient describes configuration for gRPC clients

type Scheduler

type Scheduler struct {
	// How often to run a scheduler iteration.
	ScheduleRate Duration
	// How many tasks to schedule in one iteration.
	ScheduleChunk int
	// How long to wait for a node ping before marking it as dead
	NodePingTimeout Duration
	// How long to wait for node initialization before marking it dead
	NodeInitTimeout Duration
	// How long to wait before deleting a dead node from the DB.
	NodeDeadTimeout Duration
}

Scheduler contains funnel's basic scheduler configuration.

type Server

type Server struct {
	ServiceName      string
	HostName         string
	HTTPPort         string
	RPCPort          string
	BasicAuth        []BasicCredential
	DisableHTTPCache bool
}

Server describes configuration for the server.

func (Server) HTTPAddress

func (c Server) HTTPAddress() string

HTTPAddress returns the HTTP address based on HostName and HTTPPort

func (*Server) RPCAddress

func (c *Server) RPCAddress() string

RPCAddress returns the RPC address based on HostName and RPCPort

type SwiftStorage

type SwiftStorage struct {
	Disabled   bool
	UserName   string
	Password   string
	AuthURL    string
	TenantName string
	TenantID   string
	RegionName string
	// Size of chunks to use for large object creation.
	// Defaults to 500 MB if not set or set below 10 MB.
	// The max number of chunks for a single object is 1000.
	ChunkSizeBytes int64
	// The maximum number of times to retry on error.
	// Defaults to 3.
	MaxRetries int
}

SwiftStorage configures the OpenStack Swift object storage backend.

func (SwiftStorage) Valid

func (s SwiftStorage) Valid() bool

Valid validates the SwiftStorage configuration.

type Worker

type Worker struct {
	// Directory to write task files to
	WorkDir string
	// How often the worker should poll for cancel signals
	PollingRate Duration
	// How often to update stdout/stderr log fields.
	// Setting this to 0 will result in these fields being updated a single time
	// after the executor exits.
	LogUpdateRate Duration
	// Max bytes of stdout/stderr to store in the database.
	// Setting this to 0 turns off stdout/stderr logging.
	LogTailSize int64
	// Normally the worker cleans up its working directory after executing.
	// This option disables that behavior.
	LeaveWorkDir bool
	// Limit the number of concurrent downloads/uploads
	MaxParallelTransfers int
}

Worker contains worker configuration.