spark

Documentation

Index

• Constants
• type Config
  • func GetSparkConfig() *Config
  • func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet
• type Feature
• type LogConfig

Constants

const KindSparkApplication = "SparkApplication"

Types

type Config

type Config struct {
	DefaultSparkConfig    map[string]string `json:"spark-config-default" pflag:"-,Key value pairs of default spark configuration that should be applied to every SparkJob"`
	SparkHistoryServerURL string            `json:"spark-history-server-url" pflag:",URL for SparkHistory Server that each job will publish the execution history to."`
	Features              []Feature         `json:"features" pflag:"-,List of optional features supported."`
	LogConfig             LogConfig         `json:"logs" pflag:",Config for log links for spark applications."`
}

Spark-specific configs

func GetSparkConfig

func GetSparkConfig() *Config

func (Config) GetPFlagSet

func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet

GetPFlagSet will return strongly types pflags for all fields in Config and its nested types. The format of the flags is json-name.json-sub-name... etc.

type Feature

type Feature struct {
	Name        string            `json:"name"`
	SparkConfig map[string]string `json:"spark-config"`
}

Optional feature with name and corresponding spark-config to use.

type LogConfig

type LogConfig struct {
	Mixed   logs.LogConfig `json:"mixed" pflag:",Defines the log config that's not split into user/system."`
	User    logs.LogConfig `json:"user" pflag:",Defines the log config for user logs."`
	System  logs.LogConfig `json:"system" pflag:",Defines the log config for system logs."`
	AllUser logs.LogConfig `json:"all-user" pflag:",All user logs across driver and executors."`
}