sparkonk8s

Documentation

Index

• Constants
• func CreateApiGatewayIngress(clientset *kubernetes.Clientset, namespace string, ingressName string, ...) error
• func CreateApiGatewayService(clientset *kubernetes.Clientset, namespace string, serviceName string, ...)
• func CreateInstanceIAMRole(topology SparkTopology) string
• func CreateSparkServiceAccount(clientset *kubernetes.Clientset, sparkOperatorNamespace string, ...)
• func DeployNginxIngressController(commandEnvironment framework.CommandEnvironment, topology SparkTopology) map[string]interface{}
• func DeploySparkOperator(commandEnvironment framework.CommandEnvironment, topology SparkTopology)
• func InstallSparkOperatorHelm(commandEnvironment framework.CommandEnvironment, topology SparkTopology)
• func UpdateSparkTopologyByS3BucketName(topology *SparkTopology, s3BucketName string)
• type NginxIngress
• type SparkApiGateway
• type SparkOperator
• type SparkTemplateData
  • func CreateSparkTemplateData(data framework.TemplateData) SparkTemplateData
• type SparkTopology
  • func CreateDefaultSparkTopology(namePrefix string, s3BucketName string) SparkTopology
  • func (t *SparkTopology) GetKind() string
  • func (t *SparkTopology) ToString() string
• type SparkTopologySpec
• type TopologyHandler
  • func (t *TopologyHandler) Generate() (framework.Topology, error)
  • func (t *TopologyHandler) Install(topology framework.Topology) (framework.DeploymentOutput, error)
  • func (t *TopologyHandler) Parse(yamlContent []byte) (framework.Topology, error)
  • func (t *TopologyHandler) PrintUsageExample(topology framework.Topology, deploymentOutput framework.DeploymentOutput)
  • func (t *TopologyHandler) Resolve(topology framework.Topology, data framework.TemplateData) (framework.Topology, error)
  • func (t *TopologyHandler) Uninstall(topology framework.Topology) (framework.DeploymentOutput, error)

Constants

const (
	ToBeReplacedS3BucketName            = "todo_use_your_own_bucket_name"
	DefaultApiUserName                  = "user1"
	DefaultInstanceType                 = "t3.large"
	DefaultNodeGroupSize                = 3
	DefaultOperatorImageRepository      = "ghcr.io/datapunchorg/spark-on-k8s-operator"
	DefaultSparkOperatorImageTag        = "master-datapunch"
	DefaultSparkOperatorNamespace       = "spark-operator-01"
	DefaultSparkOperatorHelmInstallName = "spark-operator-01"
	DefaultSparkApplicationNamespace    = "spark-01"
	DefaultNginxIngressHelmInstallName  = "ingress-nginx"
	DefaultNginxIngressNamespace        = "ingress-nginx"
	DefaultNginxEnableHttp              = true
	DefaultNginxEnableHttps             = true

	KindSparkTopology = "SparkOnK8s"

	FieldMaskValue = "***"

	CmdEnvHelmExecutable         = "helmExecutable"
	CmdEnvWithMinikube           = "withMinikube"
	CmdEnvNginxHelmChart         = "nginxHelmChart"
	CmdEnvSparkOperatorHelmChart = "sparkOperatorHelmChart"
	CmdEnvKubeConfig             = "kubeConfig"

	DefaultVersion        = "datapunch.org/v1alpha1"
	DefaultRegion         = "us-west-1"
	DefaultNamePrefix     = "my"
	DefaultHelmExecutable = "helm"
)

Functions

func CreateApiGatewayIngress

func CreateApiGatewayIngress(clientset *kubernetes.Clientset, namespace string, ingressName string, serviceName string) error

func CreateApiGatewayService

func CreateApiGatewayService(clientset *kubernetes.Clientset, namespace string, serviceName string, instanceName string)

func CreateInstanceIAMRole

func CreateInstanceIAMRole(topology SparkTopology) string

func CreateSparkServiceAccount

func CreateSparkServiceAccount(clientset *kubernetes.Clientset, sparkOperatorNamespace string, sparkApplicationNamespace string, sparkServiceAccountName string)

func DeployNginxIngressController

func DeployNginxIngressController(commandEnvironment framework.CommandEnvironment, topology SparkTopology) map[string]interface{}

TODO remove log.Fatalf

func DeploySparkOperator

func DeploySparkOperator(commandEnvironment framework.CommandEnvironment, topology SparkTopology)

func InstallSparkOperatorHelm

func InstallSparkOperatorHelm(commandEnvironment framework.CommandEnvironment, topology SparkTopology)

TODO remove log.Fatalf

func UpdateSparkTopologyByS3BucketName

func UpdateSparkTopologyByS3BucketName(topology *SparkTopology, s3BucketName string)

Types

type NginxIngress

type NginxIngress struct {
	HelmInstallName string `json:"helmInstallName" yaml:"helmInstallName"`
	Namespace       string `json:"namespace" yaml:"namespace"`
	EnableHttp      bool   `json:"enableHttp" yaml:"enableHttp"`
	EnableHttps     bool   `json:"enableHttps" yaml:"enableHttps"`
}

type SparkApiGateway

type SparkApiGateway struct {
	UserName     string `json:"userName" yaml:"userName"`
	UserPassword string `json:"userPassword" yaml:"userPassword"`
}

type SparkOperator

type SparkOperator struct {
	HelmInstallName           string `json:"helmInstallName" yaml:"helmInstallName"`
	Namespace                 string `json:"namespace" yaml:"namespace"`
	ImageRepository           string `json:"imageRepository" yaml:"imageRepository"`
	ImageTag                  string `json:"imageTag" yaml:"imageTag"`
	SparkApplicationNamespace string `json:"sparkApplicationNamespace" yaml:"sparkApplicationNamespace"`
}

type SparkTemplateData

type SparkTemplateData struct {
	framework.TemplateDataWithRegion
}

func CreateSparkTemplateData

func CreateSparkTemplateData(data framework.TemplateData) SparkTemplateData

type SparkTopology

type SparkTopology struct {
	ApiVersion string                     `json:"apiVersion" yaml:"apiVersion"`
	Kind       string                     `json:"kind" yaml:"kind"`
	Metadata   framework.TopologyMetadata `json:"metadata"`
	Spec       SparkTopologySpec          `json:"spec"`
}

func CreateDefaultSparkTopology

func CreateDefaultSparkTopology(namePrefix string, s3BucketName string) SparkTopology

func (*SparkTopology) GetKind

func (t *SparkTopology) GetKind() string

func (*SparkTopology) ToString

func (t *SparkTopology) ToString() string

type SparkTopologySpec

type SparkTopologySpec struct {
	NamePrefix    string               `json:"namePrefix" yaml:"namePrefix"`
	Region        string               `json:"region"`
	VpcId         string               `json:"vpcId" yaml:"vpcId"`
	S3BucketName  string               `json:"s3BucketName" yaml:"s3BucketName"`
	S3Policy      resource.IAMPolicy   `json:"s3Policy" yaml:"s3Policy"`
	EKS           resource.EKSCluster  `json:"eks" yaml:"eks"`
	NodeGroups    []resource.NodeGroup `json:"nodeGroups" yaml:"nodeGroups"`
	SparkOperator SparkOperator        `json:"sparkOperator" yaml:"sparkOperator"`
	NginxIngress  NginxIngress         `json:"nginxIngress" yaml:"nginxIngress"`
	ApiGateway    SparkApiGateway      `json:"apiGateway" yaml:"apiGateway"`
}

type TopologyHandler

type TopologyHandler struct {
}

func (*TopologyHandler) Generate

func (t *TopologyHandler) Generate() (framework.Topology, error)

func (*TopologyHandler) Install

func (t *TopologyHandler) Install(topology framework.Topology) (framework.DeploymentOutput, error)

func (*TopologyHandler) Parse

func (t *TopologyHandler) Parse(yamlContent []byte) (framework.Topology, error)

func (*TopologyHandler) PrintUsageExample

func (t *TopologyHandler) PrintUsageExample(topology framework.Topology, deploymentOutput framework.DeploymentOutput)

func (*TopologyHandler) Resolve

func (t *TopologyHandler) Resolve(topology framework.Topology, data framework.TemplateData) (framework.Topology, error)

func (*TopologyHandler) Uninstall

func (t *TopologyHandler) Uninstall(topology framework.Topology) (framework.DeploymentOutput, error)